brakeman 5.0.2 → 5.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +6 -0
  3. data/bundle/load.rb +0 -1
  4. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby20_parser.rb +278 -273
  5. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby20_parser.y +3 -0
  6. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby21_parser.rb +291 -286
  7. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby21_parser.y +3 -0
  8. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby22_parser.rb +297 -292
  9. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby22_parser.y +3 -0
  10. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby23_parser.rb +295 -290
  11. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby23_parser.y +3 -0
  12. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby24_parser.rb +296 -291
  13. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby24_parser.y +3 -0
  14. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby25_parser.rb +297 -292
  15. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby25_parser.y +3 -0
  16. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby26_parser.rb +301 -296
  17. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby26_parser.y +3 -0
  18. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby27_parser.rb +2528 -2480
  19. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby27_parser.y +26 -0
  20. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.rb +2528 -2480
  21. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.y +26 -0
  22. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby_parser.yy +30 -0
  23. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/tools/ripper.rb +1 -1
  24. data/lib/brakeman.rb +0 -4
  25. data/lib/brakeman/checks/check_detailed_exceptions.rb +1 -1
  26. data/lib/brakeman/checks/check_evaluation.rb +1 -1
  27. data/lib/brakeman/checks/check_sql.rb +2 -15
  28. data/lib/brakeman/checks/check_verb_confusion.rb +1 -1
  29. data/lib/brakeman/file_parser.rb +14 -36
  30. data/lib/brakeman/options.rb +1 -1
  31. data/lib/brakeman/processors/alias_processor.rb +7 -52
  32. data/lib/brakeman/processors/controller_alias_processor.rb +43 -6
  33. data/lib/brakeman/processors/lib/call_conversion_helper.rb +0 -10
  34. data/lib/brakeman/processors/library_processor.rb +0 -9
  35. data/lib/brakeman/report.rb +1 -4
  36. data/lib/brakeman/report/ignore/interactive.rb +1 -1
  37. data/lib/brakeman/scanner.rb +0 -3
  38. data/lib/brakeman/tracker.rb +4 -33
  39. data/lib/brakeman/tracker/collection.rb +5 -27
  40. data/lib/brakeman/util.rb +0 -8
  41. data/lib/brakeman/version.rb +1 -1
  42. metadata +2 -8
  43. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/MIT-LICENSE.txt +0 -20
  44. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel.rb +0 -523
  45. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/processor_count.rb +0 -42
  46. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/version.rb +0 -3
  47. data/lib/brakeman/report/report_github.rb +0 -31
  48. data/lib/brakeman/tracker/method_info.rb +0 -29
@@ -76,8 +76,6 @@ module Brakeman
76
76
 
77
77
  #Have to do this because first element is :array and we have to skip it
78
78
  array[1..-1][index] or original_exp
79
- elsif all_literals? array
80
- safe_literal(array.line)
81
79
  else
82
80
  original_exp
83
81
  end
@@ -94,13 +92,5 @@ module Brakeman
94
92
  original_exp
95
93
  end
96
94
  end
97
-
98
- def hash_values_at hash, keys
99
- values = keys.map do |key|
100
- process_hash_access hash, key
101
- end
102
-
103
- Sexp.new(:array).concat(values).line(hash.line)
104
- end
105
95
  end
106
96
  end
@@ -54,15 +54,6 @@ class Brakeman::LibraryProcessor < Brakeman::BaseProcessor
54
54
 
55
55
  def process_call exp
56
56
  if process_call_defn? exp
57
- exp
58
- elsif @current_method.nil? and exp.target.nil? and (@current_class or @current_module)
59
- # Methods called inside class / module
60
- case exp.method
61
- when :include
62
- module_name = class_name(exp.first_arg)
63
- (@current_class || @current_module).add_include module_name
64
- end
65
-
66
57
  exp
67
58
  else
68
59
  process_default exp
@@ -6,7 +6,7 @@ require 'brakeman/report/report_base'
6
6
  class Brakeman::Report
7
7
  attr_reader :tracker
8
8
 
9
- VALID_FORMATS = [:to_html, :to_pdf, :to_csv, :to_json, :to_tabs, :to_hash, :to_s, :to_markdown, :to_codeclimate, :to_plain, :to_text, :to_junit, :to_github]
9
+ VALID_FORMATS = [:to_html, :to_pdf, :to_csv, :to_json, :to_tabs, :to_hash, :to_s, :to_markdown, :to_codeclimate, :to_plain, :to_text, :to_junit]
10
10
 
11
11
  def initialize tracker
12
12
  @app_tree = tracker.app_tree
@@ -48,9 +48,6 @@ class Brakeman::Report
48
48
  when :to_sonar
49
49
  require_report 'sonar'
50
50
  Brakeman::Report::Sonar
51
- when :to_github
52
- require_report 'github'
53
- Brakeman::Report::Github
54
51
  else
55
52
  raise "Invalid format: #{format}. Should be one of #{VALID_FORMATS.inspect}"
56
53
  end
@@ -62,7 +62,7 @@ module Brakeman
62
62
  process_warnings
63
63
  end
64
64
 
65
- m.choice "Inspect new warnings" do
65
+ m.choice "Hide previously ignored warnings" do
66
66
  @skip_ignored = true
67
67
  pre_show_help
68
68
  process_warnings
@@ -353,9 +353,6 @@ class Brakeman::Scanner
353
353
  def parse_ruby_file file
354
354
  fp = Brakeman::FileParser.new(tracker.app_tree, tracker.options[:parser_timeout])
355
355
  fp.parse_ruby(file.read, file)
356
- rescue Exception => e
357
- tracker.error(e)
358
- nil
359
356
  end
360
357
  end
361
358
 
@@ -35,7 +35,6 @@ class Brakeman::Tracker
35
35
  #class they are.
36
36
  @models = {}
37
37
  @models[UNKNOWN_MODEL] = Brakeman::Model.new(UNKNOWN_MODEL, nil, @app_tree.file_path("NOT_REAL.rb"), nil, self)
38
- @method_cache = {}
39
38
  @routes = {}
40
39
  @initializers = {}
41
40
  @errors = []
@@ -100,8 +99,8 @@ class Brakeman::Tracker
100
99
  classes.each do |set|
101
100
  set.each do |set_name, collection|
102
101
  collection.each_method do |method_name, definition|
103
- src = definition.src
104
- yield src, set_name, method_name, definition.file
102
+ src = definition[:src]
103
+ yield src, set_name, method_name, definition[:file]
105
104
  end
106
105
  end
107
106
  end
@@ -221,34 +220,6 @@ class Brakeman::Tracker
221
220
  nil
222
221
  end
223
222
 
224
- def find_method method_name, class_name, method_type = :instance
225
- return nil unless method_name.is_a? Symbol
226
-
227
- klass = find_class(class_name)
228
- return nil unless klass
229
-
230
- cache_key = [klass, method_name, method_type]
231
-
232
- if method = @method_cache[cache_key]
233
- return method
234
- end
235
-
236
- if method = klass.get_method(method_name, method_type)
237
- return method
238
- else
239
- # Check modules included for method definition
240
- # TODO: only for instance methods, otherwise check extends!
241
- klass.includes.each do |included_name|
242
- if method = find_method(method_name, included_name, method_type)
243
- return (@method_cache[cache_key] = method)
244
- end
245
- end
246
-
247
- # Not in any included modules, check the parent
248
- @method_cache[cache_key] = find_method(method_name, klass.parent)
249
- end
250
- end
251
-
252
223
  def index_call_sites
253
224
  finder = Brakeman::FindAllCalls.new self
254
225
 
@@ -314,8 +285,8 @@ class Brakeman::Tracker
314
285
  method_sets.each do |set|
315
286
  set.each do |set_name, info|
316
287
  info.each_method do |method_name, definition|
317
- src = definition.src
318
- finder.process_source src, :class => set_name, :method => method_name, :file => definition.file
288
+ src = definition[:src]
289
+ finder.process_source src, :class => set_name, :method => method_name, :file => definition[:file]
319
290
  end
320
291
  end
321
292
  end
@@ -1,5 +1,4 @@
1
1
  require 'brakeman/util'
2
- require 'brakeman/tracker/method_info'
3
2
 
4
3
  module Brakeman
5
4
  class Collection
@@ -14,7 +13,6 @@ module Brakeman
14
13
  @src = {}
15
14
  @includes = []
16
15
  @methods = { :public => {}, :private => {}, :protected => {} }
17
- @class_methods = {}
18
16
  @options = {}
19
17
  @tracker = tracker
20
18
 
@@ -48,16 +46,11 @@ module Brakeman
48
46
  end
49
47
 
50
48
  def add_method visibility, name, src, file_name
51
- meth_info = Brakeman::MethodInfo.new(name, src, self, file_name)
52
-
53
49
  if src.node_type == :defs
54
- @class_methods[name] = meth_info
55
-
56
- # TODO fix this weirdness
57
50
  name = :"#{src[1]}.#{name}"
58
51
  end
59
52
 
60
- @methods[visibility][name] = meth_info
53
+ @methods[visibility][name] = { :src => src, :file => file_name }
61
54
  end
62
55
 
63
56
  def each_method
@@ -68,31 +61,16 @@ module Brakeman
68
61
  end
69
62
  end
70
63
 
71
- def get_method name, type = :instance
72
- case type
73
- when :class
74
- get_class_method name
75
- when :instance
76
- get_instance_method name
77
- else
78
- raise "Unexpected method type: #{type.inspect}"
79
- end
80
- end
81
-
82
- def get_instance_method name
83
- @methods.each do |_vis, meths|
84
- if meths[name]
85
- return meths[name]
64
+ def get_method name
65
+ each_method do |n, info|
66
+ if n == name
67
+ return info
86
68
  end
87
69
  end
88
70
 
89
71
  nil
90
72
  end
91
73
 
92
- def get_class_method name
93
- @class_methods[name]
94
- end
95
-
96
74
  def file
97
75
  @files.first
98
76
  end
data/lib/brakeman/util.rb CHANGED
@@ -142,14 +142,6 @@ module Brakeman::Util
142
142
  nil
143
143
  end
144
144
 
145
- def hash_values hash
146
- values = hash.each_sexp.each_slice(2).map do |_, value|
147
- value
148
- end
149
-
150
- Sexp.new(:array).concat(values).line(hash.line)
151
- end
152
-
153
145
  #These are never modified
154
146
  PARAMS_SEXP = Sexp.new(:params)
155
147
  SESSION_SEXP = Sexp.new(:session)
@@ -1,3 +1,3 @@
1
1
  module Brakeman
2
- Version = "5.0.2"
2
+ Version = "5.0.4"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: brakeman
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.0.2
4
+ version: 5.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Justin Collins
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-06-07 00:00:00.000000000 Z
11
+ date: 2021-06-08 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Brakeman detects security vulnerabilities in Ruby on Rails applications
14
14
  via static analysis.
@@ -132,10 +132,6 @@ files:
132
132
  - bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/terminal/unix_stty.rb
133
133
  - bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/version.rb
134
134
  - bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/wrapper.rb
135
- - bundle/ruby/2.7.0/gems/parallel-1.20.1/MIT-LICENSE.txt
136
- - bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel.rb
137
- - bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/processor_count.rb
138
- - bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/version.rb
139
135
  - bundle/ruby/2.7.0/gems/rexml-3.2.5/LICENSE.txt
140
136
  - bundle/ruby/2.7.0/gems/rexml-3.2.5/NEWS.md
141
137
  - bundle/ruby/2.7.0/gems/rexml-3.2.5/README.md
@@ -572,7 +568,6 @@ files:
572
568
  - lib/brakeman/report/report_base.rb
573
569
  - lib/brakeman/report/report_codeclimate.rb
574
570
  - lib/brakeman/report/report_csv.rb
575
- - lib/brakeman/report/report_github.rb
576
571
  - lib/brakeman/report/report_hash.rb
577
572
  - lib/brakeman/report/report_html.rb
578
573
  - lib/brakeman/report/report_json.rb
@@ -602,7 +597,6 @@ files:
602
597
  - lib/brakeman/tracker/constants.rb
603
598
  - lib/brakeman/tracker/controller.rb
604
599
  - lib/brakeman/tracker/library.rb
605
- - lib/brakeman/tracker/method_info.rb
606
600
  - lib/brakeman/tracker/model.rb
607
601
  - lib/brakeman/tracker/template.rb
608
602
  - lib/brakeman/util.rb
@@ -1,20 +0,0 @@
1
- Copyright (C) 2013 Michael Grosser <michael@grosser.it>
2
-
3
- Permission is hereby granted, free of charge, to any person obtaining
4
- a copy of this software and associated documentation files (the
5
- "Software"), to deal in the Software without restriction, including
6
- without limitation the rights to use, copy, modify, merge, publish,
7
- distribute, sublicense, and/or sell copies of the Software, and to
8
- permit persons to whom the Software is furnished to do so, subject to
9
- the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be
12
- included in all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -1,523 +0,0 @@
1
- require 'rbconfig'
2
- require 'parallel/version'
3
- require 'parallel/processor_count'
4
-
5
- module Parallel
6
- extend ProcessorCount
7
-
8
- Stop = Object.new.freeze
9
-
10
- class DeadWorker < StandardError
11
- end
12
-
13
- class Break < StandardError
14
- attr_reader :value
15
- def initialize(value = nil)
16
- @value = value
17
- end
18
- end
19
-
20
- class Kill < Break
21
- end
22
-
23
- class UndumpableException < StandardError
24
- attr_reader :backtrace
25
- def initialize(original)
26
- super "#{original.class}: #{original.message}"
27
- @backtrace = original.backtrace
28
- end
29
- end
30
-
31
- class ExceptionWrapper
32
- attr_reader :exception
33
- def initialize(exception)
34
- # Remove the bindings stack added by the better_errors gem,
35
- # because it cannot be marshalled
36
- if exception.instance_variable_defined? :@__better_errors_bindings_stack
37
- exception.send :remove_instance_variable, :@__better_errors_bindings_stack
38
- end
39
-
40
- @exception =
41
- begin
42
- Marshal.dump(exception) && exception
43
- rescue
44
- UndumpableException.new(exception)
45
- end
46
- end
47
- end
48
-
49
- class Worker
50
- attr_reader :pid, :read, :write
51
- attr_accessor :thread
52
- def initialize(read, write, pid)
53
- @read, @write, @pid = read, write, pid
54
- end
55
-
56
- def stop
57
- close_pipes
58
- wait # if it goes zombie, rather wait here to be able to debug
59
- end
60
-
61
- # might be passed to started_processes and simultaneously closed by another thread
62
- # when running in isolation mode, so we have to check if it is closed before closing
63
- def close_pipes
64
- read.close unless read.closed?
65
- write.close unless write.closed?
66
- end
67
-
68
- def work(data)
69
- begin
70
- Marshal.dump(data, write)
71
- rescue Errno::EPIPE
72
- raise DeadWorker
73
- end
74
-
75
- result = begin
76
- Marshal.load(read)
77
- rescue EOFError
78
- raise DeadWorker
79
- end
80
- raise result.exception if ExceptionWrapper === result
81
- result
82
- end
83
-
84
- private
85
-
86
- def wait
87
- Process.wait(pid)
88
- rescue Interrupt
89
- # process died
90
- end
91
- end
92
-
93
- class JobFactory
94
- def initialize(source, mutex)
95
- @lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
96
- @source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
97
- @mutex = mutex
98
- @index = -1
99
- @stopped = false
100
- end
101
-
102
- def next
103
- if producer?
104
- # - index and item stay in sync
105
- # - do not call lambda after it has returned Stop
106
- item, index = @mutex.synchronize do
107
- return if @stopped
108
- item = @lambda.call
109
- @stopped = (item == Stop)
110
- return if @stopped
111
- [item, @index += 1]
112
- end
113
- else
114
- index = @mutex.synchronize { @index += 1 }
115
- return if index >= size
116
- item = @source[index]
117
- end
118
- [item, index]
119
- end
120
-
121
- def size
122
- if producer?
123
- Float::INFINITY
124
- else
125
- @source.size
126
- end
127
- end
128
-
129
- # generate item that is sent to workers
130
- # just index is faster + less likely to blow up with unserializable errors
131
- def pack(item, index)
132
- producer? ? [item, index] : index
133
- end
134
-
135
- # unpack item that is sent to workers
136
- def unpack(data)
137
- producer? ? data : [@source[data], data]
138
- end
139
-
140
- private
141
-
142
- def producer?
143
- @lambda
144
- end
145
-
146
- def queue_wrapper(array)
147
- array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
148
- end
149
- end
150
-
151
- class UserInterruptHandler
152
- INTERRUPT_SIGNAL = :SIGINT
153
-
154
- class << self
155
- # kill all these pids or threads if user presses Ctrl+c
156
- def kill_on_ctrl_c(pids, options)
157
- @to_be_killed ||= []
158
- old_interrupt = nil
159
- signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
160
-
161
- if @to_be_killed.empty?
162
- old_interrupt = trap_interrupt(signal) do
163
- $stderr.puts 'Parallel execution interrupted, exiting ...'
164
- @to_be_killed.flatten.each { |pid| kill(pid) }
165
- end
166
- end
167
-
168
- @to_be_killed << pids
169
-
170
- yield
171
- ensure
172
- @to_be_killed.pop # do not kill pids that could be used for new processes
173
- restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
174
- end
175
-
176
- def kill(thing)
177
- Process.kill(:KILL, thing)
178
- rescue Errno::ESRCH
179
- # some linux systems already automatically killed the children at this point
180
- # so we just ignore them not being there
181
- end
182
-
183
- private
184
-
185
- def trap_interrupt(signal)
186
- old = Signal.trap signal, 'IGNORE'
187
-
188
- Signal.trap signal do
189
- yield
190
- if !old || old == "DEFAULT"
191
- raise Interrupt
192
- else
193
- old.call
194
- end
195
- end
196
-
197
- old
198
- end
199
-
200
- def restore_interrupt(old, signal)
201
- Signal.trap signal, old
202
- end
203
- end
204
- end
205
-
206
- class << self
207
- def in_threads(options={:count => 2})
208
- threads = []
209
- count, _ = extract_count_from_options(options)
210
-
211
- Thread.handle_interrupt(Exception => :never) do
212
- begin
213
- Thread.handle_interrupt(Exception => :immediate) do
214
- count.times do |i|
215
- threads << Thread.new { yield(i) }
216
- end
217
- threads.map(&:value)
218
- end
219
- ensure
220
- threads.each(&:kill)
221
- end
222
- end
223
- end
224
-
225
- def in_processes(options = {}, &block)
226
- count, options = extract_count_from_options(options)
227
- count ||= processor_count
228
- map(0...count, options.merge(:in_processes => count), &block)
229
- end
230
-
231
- def each(array, options={}, &block)
232
- map(array, options.merge(:preserve_results => false), &block)
233
- end
234
-
235
- def any?(*args, &block)
236
- raise "You must provide a block when calling #any?" if block.nil?
237
- !each(*args) { |*a| raise Kill if block.call(*a) }
238
- end
239
-
240
- def all?(*args, &block)
241
- raise "You must provide a block when calling #all?" if block.nil?
242
- !!each(*args) { |*a| raise Kill unless block.call(*a) }
243
- end
244
-
245
- def each_with_index(array, options={}, &block)
246
- each(array, options.merge(:with_index => true), &block)
247
- end
248
-
249
- def map(source, options = {}, &block)
250
- options = options.dup
251
- options[:mutex] = Mutex.new
252
-
253
- if options[:in_processes] && options[:in_threads]
254
- raise ArgumentError.new("Please specify only one of `in_processes` or `in_threads`.")
255
- elsif RUBY_PLATFORM =~ /java/ and not options[:in_processes]
256
- method = :in_threads
257
- size = options[method] || processor_count
258
- elsif options[:in_threads]
259
- method = :in_threads
260
- size = options[method]
261
- else
262
- method = :in_processes
263
- if Process.respond_to?(:fork)
264
- size = options[method] || processor_count
265
- else
266
- warn "Process.fork is not supported by this Ruby"
267
- size = 0
268
- end
269
- end
270
-
271
- job_factory = JobFactory.new(source, options[:mutex])
272
- size = [job_factory.size, size].min
273
-
274
- options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
275
- add_progress_bar!(job_factory, options)
276
-
277
- result =
278
- if size == 0
279
- work_direct(job_factory, options, &block)
280
- elsif method == :in_threads
281
- work_in_threads(job_factory, options.merge(:count => size), &block)
282
- else
283
- work_in_processes(job_factory, options.merge(:count => size), &block)
284
- end
285
-
286
- return result.value if result.is_a?(Break)
287
- raise result if result.is_a?(Exception)
288
- options[:return_results] ? result : source
289
- end
290
-
291
- def map_with_index(array, options={}, &block)
292
- map(array, options.merge(:with_index => true), &block)
293
- end
294
-
295
- def flat_map(*args, &block)
296
- map(*args, &block).flatten(1)
297
- end
298
-
299
- def worker_number
300
- Thread.current[:parallel_worker_number]
301
- end
302
-
303
- # TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
304
- def worker_number=(worker_num)
305
- Thread.current[:parallel_worker_number] = worker_num
306
- end
307
-
308
- private
309
-
310
- def add_progress_bar!(job_factory, options)
311
- if progress_options = options[:progress]
312
- raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
313
- require 'ruby-progressbar'
314
-
315
- if progress_options == true
316
- progress_options = { title: "Progress" }
317
- elsif progress_options.respond_to? :to_str
318
- progress_options = { title: progress_options.to_str }
319
- end
320
-
321
- progress_options = {
322
- total: job_factory.size,
323
- format: '%t |%E | %B | %a'
324
- }.merge(progress_options)
325
-
326
- progress = ProgressBar.create(progress_options)
327
- old_finish = options[:finish]
328
- options[:finish] = lambda do |item, i, result|
329
- old_finish.call(item, i, result) if old_finish
330
- progress.increment
331
- end
332
- end
333
- end
334
-
335
- def work_direct(job_factory, options, &block)
336
- self.worker_number = 0
337
- results = []
338
- exception = nil
339
- begin
340
- while set = job_factory.next
341
- item, index = set
342
- results << with_instrumentation(item, index, options) do
343
- call_with_index(item, index, options, &block)
344
- end
345
- end
346
- rescue
347
- exception = $!
348
- end
349
- exception || results
350
- ensure
351
- self.worker_number = nil
352
- end
353
-
354
- def work_in_threads(job_factory, options, &block)
355
- raise "interrupt_signal is no longer supported for threads" if options[:interrupt_signal]
356
- results = []
357
- results_mutex = Mutex.new # arrays are not thread-safe on jRuby
358
- exception = nil
359
-
360
- in_threads(options) do |worker_num|
361
- self.worker_number = worker_num
362
- # as long as there are more jobs, work on one of them
363
- while !exception && set = job_factory.next
364
- begin
365
- item, index = set
366
- result = with_instrumentation item, index, options do
367
- call_with_index(item, index, options, &block)
368
- end
369
- results_mutex.synchronize { results[index] = result }
370
- rescue
371
- exception = $!
372
- end
373
- end
374
- end
375
-
376
- exception || results
377
- end
378
-
379
- def work_in_processes(job_factory, options, &blk)
380
- workers = create_workers(job_factory, options, &blk)
381
- results = []
382
- results_mutex = Mutex.new # arrays are not thread-safe
383
- exception = nil
384
-
385
- UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
386
- in_threads(options) do |i|
387
- worker = workers[i]
388
- worker.thread = Thread.current
389
- worked = false
390
-
391
- begin
392
- loop do
393
- break if exception
394
- item, index = job_factory.next
395
- break unless index
396
-
397
- if options[:isolation]
398
- worker = replace_worker(job_factory, workers, i, options, blk) if worked
399
- worked = true
400
- worker.thread = Thread.current
401
- end
402
-
403
- begin
404
- result = with_instrumentation item, index, options do
405
- worker.work(job_factory.pack(item, index))
406
- end
407
- results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
408
- rescue StandardError => e
409
- exception = e
410
- if Kill === exception
411
- (workers - [worker]).each do |w|
412
- w.thread.kill if w.thread
413
- UserInterruptHandler.kill(w.pid)
414
- end
415
- end
416
- end
417
- end
418
- ensure
419
- worker.stop
420
- end
421
- end
422
- end
423
- exception || results
424
- end
425
-
426
- def replace_worker(job_factory, workers, i, options, blk)
427
- options[:mutex].synchronize do
428
- # old worker is no longer used ... stop it
429
- worker = workers[i]
430
- worker.stop
431
-
432
- # create a new replacement worker
433
- running = workers - [worker]
434
- workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk)
435
- end
436
- end
437
-
438
- def create_workers(job_factory, options, &block)
439
- workers = []
440
- Array.new(options[:count]).each_with_index do |_, i|
441
- workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block)
442
- end
443
- workers
444
- end
445
-
446
- def worker(job_factory, options, &block)
447
- child_read, parent_write = IO.pipe
448
- parent_read, child_write = IO.pipe
449
-
450
- pid = Process.fork do
451
- self.worker_number = options[:worker_number]
452
-
453
- begin
454
- options.delete(:started_workers).each(&:close_pipes)
455
-
456
- parent_write.close
457
- parent_read.close
458
-
459
- process_incoming_jobs(child_read, child_write, job_factory, options, &block)
460
- ensure
461
- child_read.close
462
- child_write.close
463
- end
464
- end
465
-
466
- child_read.close
467
- child_write.close
468
-
469
- Worker.new(parent_read, parent_write, pid)
470
- end
471
-
472
- def process_incoming_jobs(read, write, job_factory, options, &block)
473
- until read.eof?
474
- data = Marshal.load(read)
475
- item, index = job_factory.unpack(data)
476
- result = begin
477
- call_with_index(item, index, options, &block)
478
- # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
479
- rescue NoMemoryError, SignalException, Interrupt, SystemExit
480
- raise $!
481
- rescue Exception
482
- ExceptionWrapper.new($!)
483
- end
484
- begin
485
- Marshal.dump(result, write)
486
- rescue Errno::EPIPE
487
- return # parent thread already dead
488
- end
489
- end
490
- end
491
-
492
- # options is either a Integer or a Hash with :count
493
- def extract_count_from_options(options)
494
- if options.is_a?(Hash)
495
- count = options[:count]
496
- else
497
- count = options
498
- options = {}
499
- end
500
- [count, options]
501
- end
502
-
503
- def call_with_index(item, index, options, &block)
504
- args = [item]
505
- args << index if options[:with_index]
506
- if options[:return_results]
507
- block.call(*args)
508
- else
509
- block.call(*args)
510
- nil # avoid GC overhead of passing large results around
511
- end
512
- end
513
-
514
- def with_instrumentation(item, index, options)
515
- on_start = options[:start]
516
- on_finish = options[:finish]
517
- options[:mutex].synchronize { on_start.call(item, index) } if on_start
518
- result = yield
519
- options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish
520
- result unless options[:preserve_results] == false
521
- end
522
- end
523
- end