brakeman 5.0.2 → 5.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +6 -0
- data/bundle/load.rb +0 -1
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby20_parser.rb +278 -273
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby20_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby21_parser.rb +291 -286
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby21_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby22_parser.rb +297 -292
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby22_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby23_parser.rb +295 -290
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby23_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby24_parser.rb +296 -291
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby24_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby25_parser.rb +297 -292
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby25_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby26_parser.rb +301 -296
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby26_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby27_parser.rb +2528 -2480
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby27_parser.y +26 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.rb +2528 -2480
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.y +26 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby_parser.yy +30 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/tools/ripper.rb +1 -1
- data/lib/brakeman.rb +0 -4
- data/lib/brakeman/checks/check_detailed_exceptions.rb +1 -1
- data/lib/brakeman/checks/check_evaluation.rb +1 -1
- data/lib/brakeman/checks/check_sql.rb +2 -15
- data/lib/brakeman/checks/check_verb_confusion.rb +1 -1
- data/lib/brakeman/file_parser.rb +14 -36
- data/lib/brakeman/options.rb +1 -1
- data/lib/brakeman/processors/alias_processor.rb +7 -52
- data/lib/brakeman/processors/controller_alias_processor.rb +43 -6
- data/lib/brakeman/processors/lib/call_conversion_helper.rb +0 -10
- data/lib/brakeman/processors/library_processor.rb +0 -9
- data/lib/brakeman/report.rb +1 -4
- data/lib/brakeman/report/ignore/interactive.rb +1 -1
- data/lib/brakeman/scanner.rb +0 -3
- data/lib/brakeman/tracker.rb +4 -33
- data/lib/brakeman/tracker/collection.rb +5 -27
- data/lib/brakeman/util.rb +0 -8
- data/lib/brakeman/version.rb +1 -1
- metadata +2 -8
- data/bundle/ruby/2.7.0/gems/parallel-1.20.1/MIT-LICENSE.txt +0 -20
- data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel.rb +0 -523
- data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/processor_count.rb +0 -42
- data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/version.rb +0 -3
- data/lib/brakeman/report/report_github.rb +0 -31
- data/lib/brakeman/tracker/method_info.rb +0 -29
@@ -76,8 +76,6 @@ module Brakeman
|
|
76
76
|
|
77
77
|
#Have to do this because first element is :array and we have to skip it
|
78
78
|
array[1..-1][index] or original_exp
|
79
|
-
elsif all_literals? array
|
80
|
-
safe_literal(array.line)
|
81
79
|
else
|
82
80
|
original_exp
|
83
81
|
end
|
@@ -94,13 +92,5 @@ module Brakeman
|
|
94
92
|
original_exp
|
95
93
|
end
|
96
94
|
end
|
97
|
-
|
98
|
-
def hash_values_at hash, keys
|
99
|
-
values = keys.map do |key|
|
100
|
-
process_hash_access hash, key
|
101
|
-
end
|
102
|
-
|
103
|
-
Sexp.new(:array).concat(values).line(hash.line)
|
104
|
-
end
|
105
95
|
end
|
106
96
|
end
|
@@ -54,15 +54,6 @@ class Brakeman::LibraryProcessor < Brakeman::BaseProcessor
|
|
54
54
|
|
55
55
|
def process_call exp
|
56
56
|
if process_call_defn? exp
|
57
|
-
exp
|
58
|
-
elsif @current_method.nil? and exp.target.nil? and (@current_class or @current_module)
|
59
|
-
# Methods called inside class / module
|
60
|
-
case exp.method
|
61
|
-
when :include
|
62
|
-
module_name = class_name(exp.first_arg)
|
63
|
-
(@current_class || @current_module).add_include module_name
|
64
|
-
end
|
65
|
-
|
66
57
|
exp
|
67
58
|
else
|
68
59
|
process_default exp
|
data/lib/brakeman/report.rb
CHANGED
@@ -6,7 +6,7 @@ require 'brakeman/report/report_base'
|
|
6
6
|
class Brakeman::Report
|
7
7
|
attr_reader :tracker
|
8
8
|
|
9
|
-
VALID_FORMATS = [:to_html, :to_pdf, :to_csv, :to_json, :to_tabs, :to_hash, :to_s, :to_markdown, :to_codeclimate, :to_plain, :to_text, :to_junit
|
9
|
+
VALID_FORMATS = [:to_html, :to_pdf, :to_csv, :to_json, :to_tabs, :to_hash, :to_s, :to_markdown, :to_codeclimate, :to_plain, :to_text, :to_junit]
|
10
10
|
|
11
11
|
def initialize tracker
|
12
12
|
@app_tree = tracker.app_tree
|
@@ -48,9 +48,6 @@ class Brakeman::Report
|
|
48
48
|
when :to_sonar
|
49
49
|
require_report 'sonar'
|
50
50
|
Brakeman::Report::Sonar
|
51
|
-
when :to_github
|
52
|
-
require_report 'github'
|
53
|
-
Brakeman::Report::Github
|
54
51
|
else
|
55
52
|
raise "Invalid format: #{format}. Should be one of #{VALID_FORMATS.inspect}"
|
56
53
|
end
|
data/lib/brakeman/scanner.rb
CHANGED
data/lib/brakeman/tracker.rb
CHANGED
@@ -35,7 +35,6 @@ class Brakeman::Tracker
|
|
35
35
|
#class they are.
|
36
36
|
@models = {}
|
37
37
|
@models[UNKNOWN_MODEL] = Brakeman::Model.new(UNKNOWN_MODEL, nil, @app_tree.file_path("NOT_REAL.rb"), nil, self)
|
38
|
-
@method_cache = {}
|
39
38
|
@routes = {}
|
40
39
|
@initializers = {}
|
41
40
|
@errors = []
|
@@ -100,8 +99,8 @@ class Brakeman::Tracker
|
|
100
99
|
classes.each do |set|
|
101
100
|
set.each do |set_name, collection|
|
102
101
|
collection.each_method do |method_name, definition|
|
103
|
-
src = definition
|
104
|
-
yield src, set_name, method_name, definition
|
102
|
+
src = definition[:src]
|
103
|
+
yield src, set_name, method_name, definition[:file]
|
105
104
|
end
|
106
105
|
end
|
107
106
|
end
|
@@ -221,34 +220,6 @@ class Brakeman::Tracker
|
|
221
220
|
nil
|
222
221
|
end
|
223
222
|
|
224
|
-
def find_method method_name, class_name, method_type = :instance
|
225
|
-
return nil unless method_name.is_a? Symbol
|
226
|
-
|
227
|
-
klass = find_class(class_name)
|
228
|
-
return nil unless klass
|
229
|
-
|
230
|
-
cache_key = [klass, method_name, method_type]
|
231
|
-
|
232
|
-
if method = @method_cache[cache_key]
|
233
|
-
return method
|
234
|
-
end
|
235
|
-
|
236
|
-
if method = klass.get_method(method_name, method_type)
|
237
|
-
return method
|
238
|
-
else
|
239
|
-
# Check modules included for method definition
|
240
|
-
# TODO: only for instance methods, otherwise check extends!
|
241
|
-
klass.includes.each do |included_name|
|
242
|
-
if method = find_method(method_name, included_name, method_type)
|
243
|
-
return (@method_cache[cache_key] = method)
|
244
|
-
end
|
245
|
-
end
|
246
|
-
|
247
|
-
# Not in any included modules, check the parent
|
248
|
-
@method_cache[cache_key] = find_method(method_name, klass.parent)
|
249
|
-
end
|
250
|
-
end
|
251
|
-
|
252
223
|
def index_call_sites
|
253
224
|
finder = Brakeman::FindAllCalls.new self
|
254
225
|
|
@@ -314,8 +285,8 @@ class Brakeman::Tracker
|
|
314
285
|
method_sets.each do |set|
|
315
286
|
set.each do |set_name, info|
|
316
287
|
info.each_method do |method_name, definition|
|
317
|
-
src = definition
|
318
|
-
finder.process_source src, :class => set_name, :method => method_name, :file => definition
|
288
|
+
src = definition[:src]
|
289
|
+
finder.process_source src, :class => set_name, :method => method_name, :file => definition[:file]
|
319
290
|
end
|
320
291
|
end
|
321
292
|
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'brakeman/util'
|
2
|
-
require 'brakeman/tracker/method_info'
|
3
2
|
|
4
3
|
module Brakeman
|
5
4
|
class Collection
|
@@ -14,7 +13,6 @@ module Brakeman
|
|
14
13
|
@src = {}
|
15
14
|
@includes = []
|
16
15
|
@methods = { :public => {}, :private => {}, :protected => {} }
|
17
|
-
@class_methods = {}
|
18
16
|
@options = {}
|
19
17
|
@tracker = tracker
|
20
18
|
|
@@ -48,16 +46,11 @@ module Brakeman
|
|
48
46
|
end
|
49
47
|
|
50
48
|
def add_method visibility, name, src, file_name
|
51
|
-
meth_info = Brakeman::MethodInfo.new(name, src, self, file_name)
|
52
|
-
|
53
49
|
if src.node_type == :defs
|
54
|
-
@class_methods[name] = meth_info
|
55
|
-
|
56
|
-
# TODO fix this weirdness
|
57
50
|
name = :"#{src[1]}.#{name}"
|
58
51
|
end
|
59
52
|
|
60
|
-
@methods[visibility][name] =
|
53
|
+
@methods[visibility][name] = { :src => src, :file => file_name }
|
61
54
|
end
|
62
55
|
|
63
56
|
def each_method
|
@@ -68,31 +61,16 @@ module Brakeman
|
|
68
61
|
end
|
69
62
|
end
|
70
63
|
|
71
|
-
def get_method name
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
when :instance
|
76
|
-
get_instance_method name
|
77
|
-
else
|
78
|
-
raise "Unexpected method type: #{type.inspect}"
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
def get_instance_method name
|
83
|
-
@methods.each do |_vis, meths|
|
84
|
-
if meths[name]
|
85
|
-
return meths[name]
|
64
|
+
def get_method name
|
65
|
+
each_method do |n, info|
|
66
|
+
if n == name
|
67
|
+
return info
|
86
68
|
end
|
87
69
|
end
|
88
70
|
|
89
71
|
nil
|
90
72
|
end
|
91
73
|
|
92
|
-
def get_class_method name
|
93
|
-
@class_methods[name]
|
94
|
-
end
|
95
|
-
|
96
74
|
def file
|
97
75
|
@files.first
|
98
76
|
end
|
data/lib/brakeman/util.rb
CHANGED
@@ -142,14 +142,6 @@ module Brakeman::Util
|
|
142
142
|
nil
|
143
143
|
end
|
144
144
|
|
145
|
-
def hash_values hash
|
146
|
-
values = hash.each_sexp.each_slice(2).map do |_, value|
|
147
|
-
value
|
148
|
-
end
|
149
|
-
|
150
|
-
Sexp.new(:array).concat(values).line(hash.line)
|
151
|
-
end
|
152
|
-
|
153
145
|
#These are never modified
|
154
146
|
PARAMS_SEXP = Sexp.new(:params)
|
155
147
|
SESSION_SEXP = Sexp.new(:session)
|
data/lib/brakeman/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: brakeman
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.0.
|
4
|
+
version: 5.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Collins
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-06-
|
11
|
+
date: 2021-06-08 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Brakeman detects security vulnerabilities in Ruby on Rails applications
|
14
14
|
via static analysis.
|
@@ -132,10 +132,6 @@ files:
|
|
132
132
|
- bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/terminal/unix_stty.rb
|
133
133
|
- bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/version.rb
|
134
134
|
- bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/wrapper.rb
|
135
|
-
- bundle/ruby/2.7.0/gems/parallel-1.20.1/MIT-LICENSE.txt
|
136
|
-
- bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel.rb
|
137
|
-
- bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/processor_count.rb
|
138
|
-
- bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/version.rb
|
139
135
|
- bundle/ruby/2.7.0/gems/rexml-3.2.5/LICENSE.txt
|
140
136
|
- bundle/ruby/2.7.0/gems/rexml-3.2.5/NEWS.md
|
141
137
|
- bundle/ruby/2.7.0/gems/rexml-3.2.5/README.md
|
@@ -572,7 +568,6 @@ files:
|
|
572
568
|
- lib/brakeman/report/report_base.rb
|
573
569
|
- lib/brakeman/report/report_codeclimate.rb
|
574
570
|
- lib/brakeman/report/report_csv.rb
|
575
|
-
- lib/brakeman/report/report_github.rb
|
576
571
|
- lib/brakeman/report/report_hash.rb
|
577
572
|
- lib/brakeman/report/report_html.rb
|
578
573
|
- lib/brakeman/report/report_json.rb
|
@@ -602,7 +597,6 @@ files:
|
|
602
597
|
- lib/brakeman/tracker/constants.rb
|
603
598
|
- lib/brakeman/tracker/controller.rb
|
604
599
|
- lib/brakeman/tracker/library.rb
|
605
|
-
- lib/brakeman/tracker/method_info.rb
|
606
600
|
- lib/brakeman/tracker/model.rb
|
607
601
|
- lib/brakeman/tracker/template.rb
|
608
602
|
- lib/brakeman/util.rb
|
@@ -1,20 +0,0 @@
|
|
1
|
-
Copyright (C) 2013 Michael Grosser <michael@grosser.it>
|
2
|
-
|
3
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
-
a copy of this software and associated documentation files (the
|
5
|
-
"Software"), to deal in the Software without restriction, including
|
6
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
-
permit persons to whom the Software is furnished to do so, subject to
|
9
|
-
the following conditions:
|
10
|
-
|
11
|
-
The above copyright notice and this permission notice shall be
|
12
|
-
included in all copies or substantial portions of the Software.
|
13
|
-
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
@@ -1,523 +0,0 @@
|
|
1
|
-
require 'rbconfig'
|
2
|
-
require 'parallel/version'
|
3
|
-
require 'parallel/processor_count'
|
4
|
-
|
5
|
-
module Parallel
|
6
|
-
extend ProcessorCount
|
7
|
-
|
8
|
-
Stop = Object.new.freeze
|
9
|
-
|
10
|
-
class DeadWorker < StandardError
|
11
|
-
end
|
12
|
-
|
13
|
-
class Break < StandardError
|
14
|
-
attr_reader :value
|
15
|
-
def initialize(value = nil)
|
16
|
-
@value = value
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
class Kill < Break
|
21
|
-
end
|
22
|
-
|
23
|
-
class UndumpableException < StandardError
|
24
|
-
attr_reader :backtrace
|
25
|
-
def initialize(original)
|
26
|
-
super "#{original.class}: #{original.message}"
|
27
|
-
@backtrace = original.backtrace
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
class ExceptionWrapper
|
32
|
-
attr_reader :exception
|
33
|
-
def initialize(exception)
|
34
|
-
# Remove the bindings stack added by the better_errors gem,
|
35
|
-
# because it cannot be marshalled
|
36
|
-
if exception.instance_variable_defined? :@__better_errors_bindings_stack
|
37
|
-
exception.send :remove_instance_variable, :@__better_errors_bindings_stack
|
38
|
-
end
|
39
|
-
|
40
|
-
@exception =
|
41
|
-
begin
|
42
|
-
Marshal.dump(exception) && exception
|
43
|
-
rescue
|
44
|
-
UndumpableException.new(exception)
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
class Worker
|
50
|
-
attr_reader :pid, :read, :write
|
51
|
-
attr_accessor :thread
|
52
|
-
def initialize(read, write, pid)
|
53
|
-
@read, @write, @pid = read, write, pid
|
54
|
-
end
|
55
|
-
|
56
|
-
def stop
|
57
|
-
close_pipes
|
58
|
-
wait # if it goes zombie, rather wait here to be able to debug
|
59
|
-
end
|
60
|
-
|
61
|
-
# might be passed to started_processes and simultaneously closed by another thread
|
62
|
-
# when running in isolation mode, so we have to check if it is closed before closing
|
63
|
-
def close_pipes
|
64
|
-
read.close unless read.closed?
|
65
|
-
write.close unless write.closed?
|
66
|
-
end
|
67
|
-
|
68
|
-
def work(data)
|
69
|
-
begin
|
70
|
-
Marshal.dump(data, write)
|
71
|
-
rescue Errno::EPIPE
|
72
|
-
raise DeadWorker
|
73
|
-
end
|
74
|
-
|
75
|
-
result = begin
|
76
|
-
Marshal.load(read)
|
77
|
-
rescue EOFError
|
78
|
-
raise DeadWorker
|
79
|
-
end
|
80
|
-
raise result.exception if ExceptionWrapper === result
|
81
|
-
result
|
82
|
-
end
|
83
|
-
|
84
|
-
private
|
85
|
-
|
86
|
-
def wait
|
87
|
-
Process.wait(pid)
|
88
|
-
rescue Interrupt
|
89
|
-
# process died
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
class JobFactory
|
94
|
-
def initialize(source, mutex)
|
95
|
-
@lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
|
96
|
-
@source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
|
97
|
-
@mutex = mutex
|
98
|
-
@index = -1
|
99
|
-
@stopped = false
|
100
|
-
end
|
101
|
-
|
102
|
-
def next
|
103
|
-
if producer?
|
104
|
-
# - index and item stay in sync
|
105
|
-
# - do not call lambda after it has returned Stop
|
106
|
-
item, index = @mutex.synchronize do
|
107
|
-
return if @stopped
|
108
|
-
item = @lambda.call
|
109
|
-
@stopped = (item == Stop)
|
110
|
-
return if @stopped
|
111
|
-
[item, @index += 1]
|
112
|
-
end
|
113
|
-
else
|
114
|
-
index = @mutex.synchronize { @index += 1 }
|
115
|
-
return if index >= size
|
116
|
-
item = @source[index]
|
117
|
-
end
|
118
|
-
[item, index]
|
119
|
-
end
|
120
|
-
|
121
|
-
def size
|
122
|
-
if producer?
|
123
|
-
Float::INFINITY
|
124
|
-
else
|
125
|
-
@source.size
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
|
-
# generate item that is sent to workers
|
130
|
-
# just index is faster + less likely to blow up with unserializable errors
|
131
|
-
def pack(item, index)
|
132
|
-
producer? ? [item, index] : index
|
133
|
-
end
|
134
|
-
|
135
|
-
# unpack item that is sent to workers
|
136
|
-
def unpack(data)
|
137
|
-
producer? ? data : [@source[data], data]
|
138
|
-
end
|
139
|
-
|
140
|
-
private
|
141
|
-
|
142
|
-
def producer?
|
143
|
-
@lambda
|
144
|
-
end
|
145
|
-
|
146
|
-
def queue_wrapper(array)
|
147
|
-
array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
|
148
|
-
end
|
149
|
-
end
|
150
|
-
|
151
|
-
class UserInterruptHandler
|
152
|
-
INTERRUPT_SIGNAL = :SIGINT
|
153
|
-
|
154
|
-
class << self
|
155
|
-
# kill all these pids or threads if user presses Ctrl+c
|
156
|
-
def kill_on_ctrl_c(pids, options)
|
157
|
-
@to_be_killed ||= []
|
158
|
-
old_interrupt = nil
|
159
|
-
signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
|
160
|
-
|
161
|
-
if @to_be_killed.empty?
|
162
|
-
old_interrupt = trap_interrupt(signal) do
|
163
|
-
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
164
|
-
@to_be_killed.flatten.each { |pid| kill(pid) }
|
165
|
-
end
|
166
|
-
end
|
167
|
-
|
168
|
-
@to_be_killed << pids
|
169
|
-
|
170
|
-
yield
|
171
|
-
ensure
|
172
|
-
@to_be_killed.pop # do not kill pids that could be used for new processes
|
173
|
-
restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
|
174
|
-
end
|
175
|
-
|
176
|
-
def kill(thing)
|
177
|
-
Process.kill(:KILL, thing)
|
178
|
-
rescue Errno::ESRCH
|
179
|
-
# some linux systems already automatically killed the children at this point
|
180
|
-
# so we just ignore them not being there
|
181
|
-
end
|
182
|
-
|
183
|
-
private
|
184
|
-
|
185
|
-
def trap_interrupt(signal)
|
186
|
-
old = Signal.trap signal, 'IGNORE'
|
187
|
-
|
188
|
-
Signal.trap signal do
|
189
|
-
yield
|
190
|
-
if !old || old == "DEFAULT"
|
191
|
-
raise Interrupt
|
192
|
-
else
|
193
|
-
old.call
|
194
|
-
end
|
195
|
-
end
|
196
|
-
|
197
|
-
old
|
198
|
-
end
|
199
|
-
|
200
|
-
def restore_interrupt(old, signal)
|
201
|
-
Signal.trap signal, old
|
202
|
-
end
|
203
|
-
end
|
204
|
-
end
|
205
|
-
|
206
|
-
class << self
|
207
|
-
def in_threads(options={:count => 2})
|
208
|
-
threads = []
|
209
|
-
count, _ = extract_count_from_options(options)
|
210
|
-
|
211
|
-
Thread.handle_interrupt(Exception => :never) do
|
212
|
-
begin
|
213
|
-
Thread.handle_interrupt(Exception => :immediate) do
|
214
|
-
count.times do |i|
|
215
|
-
threads << Thread.new { yield(i) }
|
216
|
-
end
|
217
|
-
threads.map(&:value)
|
218
|
-
end
|
219
|
-
ensure
|
220
|
-
threads.each(&:kill)
|
221
|
-
end
|
222
|
-
end
|
223
|
-
end
|
224
|
-
|
225
|
-
def in_processes(options = {}, &block)
|
226
|
-
count, options = extract_count_from_options(options)
|
227
|
-
count ||= processor_count
|
228
|
-
map(0...count, options.merge(:in_processes => count), &block)
|
229
|
-
end
|
230
|
-
|
231
|
-
def each(array, options={}, &block)
|
232
|
-
map(array, options.merge(:preserve_results => false), &block)
|
233
|
-
end
|
234
|
-
|
235
|
-
def any?(*args, &block)
|
236
|
-
raise "You must provide a block when calling #any?" if block.nil?
|
237
|
-
!each(*args) { |*a| raise Kill if block.call(*a) }
|
238
|
-
end
|
239
|
-
|
240
|
-
def all?(*args, &block)
|
241
|
-
raise "You must provide a block when calling #all?" if block.nil?
|
242
|
-
!!each(*args) { |*a| raise Kill unless block.call(*a) }
|
243
|
-
end
|
244
|
-
|
245
|
-
def each_with_index(array, options={}, &block)
|
246
|
-
each(array, options.merge(:with_index => true), &block)
|
247
|
-
end
|
248
|
-
|
249
|
-
def map(source, options = {}, &block)
|
250
|
-
options = options.dup
|
251
|
-
options[:mutex] = Mutex.new
|
252
|
-
|
253
|
-
if options[:in_processes] && options[:in_threads]
|
254
|
-
raise ArgumentError.new("Please specify only one of `in_processes` or `in_threads`.")
|
255
|
-
elsif RUBY_PLATFORM =~ /java/ and not options[:in_processes]
|
256
|
-
method = :in_threads
|
257
|
-
size = options[method] || processor_count
|
258
|
-
elsif options[:in_threads]
|
259
|
-
method = :in_threads
|
260
|
-
size = options[method]
|
261
|
-
else
|
262
|
-
method = :in_processes
|
263
|
-
if Process.respond_to?(:fork)
|
264
|
-
size = options[method] || processor_count
|
265
|
-
else
|
266
|
-
warn "Process.fork is not supported by this Ruby"
|
267
|
-
size = 0
|
268
|
-
end
|
269
|
-
end
|
270
|
-
|
271
|
-
job_factory = JobFactory.new(source, options[:mutex])
|
272
|
-
size = [job_factory.size, size].min
|
273
|
-
|
274
|
-
options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
|
275
|
-
add_progress_bar!(job_factory, options)
|
276
|
-
|
277
|
-
result =
|
278
|
-
if size == 0
|
279
|
-
work_direct(job_factory, options, &block)
|
280
|
-
elsif method == :in_threads
|
281
|
-
work_in_threads(job_factory, options.merge(:count => size), &block)
|
282
|
-
else
|
283
|
-
work_in_processes(job_factory, options.merge(:count => size), &block)
|
284
|
-
end
|
285
|
-
|
286
|
-
return result.value if result.is_a?(Break)
|
287
|
-
raise result if result.is_a?(Exception)
|
288
|
-
options[:return_results] ? result : source
|
289
|
-
end
|
290
|
-
|
291
|
-
def map_with_index(array, options={}, &block)
|
292
|
-
map(array, options.merge(:with_index => true), &block)
|
293
|
-
end
|
294
|
-
|
295
|
-
def flat_map(*args, &block)
|
296
|
-
map(*args, &block).flatten(1)
|
297
|
-
end
|
298
|
-
|
299
|
-
def worker_number
|
300
|
-
Thread.current[:parallel_worker_number]
|
301
|
-
end
|
302
|
-
|
303
|
-
# TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
|
304
|
-
def worker_number=(worker_num)
|
305
|
-
Thread.current[:parallel_worker_number] = worker_num
|
306
|
-
end
|
307
|
-
|
308
|
-
private
|
309
|
-
|
310
|
-
def add_progress_bar!(job_factory, options)
|
311
|
-
if progress_options = options[:progress]
|
312
|
-
raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
|
313
|
-
require 'ruby-progressbar'
|
314
|
-
|
315
|
-
if progress_options == true
|
316
|
-
progress_options = { title: "Progress" }
|
317
|
-
elsif progress_options.respond_to? :to_str
|
318
|
-
progress_options = { title: progress_options.to_str }
|
319
|
-
end
|
320
|
-
|
321
|
-
progress_options = {
|
322
|
-
total: job_factory.size,
|
323
|
-
format: '%t |%E | %B | %a'
|
324
|
-
}.merge(progress_options)
|
325
|
-
|
326
|
-
progress = ProgressBar.create(progress_options)
|
327
|
-
old_finish = options[:finish]
|
328
|
-
options[:finish] = lambda do |item, i, result|
|
329
|
-
old_finish.call(item, i, result) if old_finish
|
330
|
-
progress.increment
|
331
|
-
end
|
332
|
-
end
|
333
|
-
end
|
334
|
-
|
335
|
-
def work_direct(job_factory, options, &block)
|
336
|
-
self.worker_number = 0
|
337
|
-
results = []
|
338
|
-
exception = nil
|
339
|
-
begin
|
340
|
-
while set = job_factory.next
|
341
|
-
item, index = set
|
342
|
-
results << with_instrumentation(item, index, options) do
|
343
|
-
call_with_index(item, index, options, &block)
|
344
|
-
end
|
345
|
-
end
|
346
|
-
rescue
|
347
|
-
exception = $!
|
348
|
-
end
|
349
|
-
exception || results
|
350
|
-
ensure
|
351
|
-
self.worker_number = nil
|
352
|
-
end
|
353
|
-
|
354
|
-
def work_in_threads(job_factory, options, &block)
|
355
|
-
raise "interrupt_signal is no longer supported for threads" if options[:interrupt_signal]
|
356
|
-
results = []
|
357
|
-
results_mutex = Mutex.new # arrays are not thread-safe on jRuby
|
358
|
-
exception = nil
|
359
|
-
|
360
|
-
in_threads(options) do |worker_num|
|
361
|
-
self.worker_number = worker_num
|
362
|
-
# as long as there are more jobs, work on one of them
|
363
|
-
while !exception && set = job_factory.next
|
364
|
-
begin
|
365
|
-
item, index = set
|
366
|
-
result = with_instrumentation item, index, options do
|
367
|
-
call_with_index(item, index, options, &block)
|
368
|
-
end
|
369
|
-
results_mutex.synchronize { results[index] = result }
|
370
|
-
rescue
|
371
|
-
exception = $!
|
372
|
-
end
|
373
|
-
end
|
374
|
-
end
|
375
|
-
|
376
|
-
exception || results
|
377
|
-
end
|
378
|
-
|
379
|
-
def work_in_processes(job_factory, options, &blk)
|
380
|
-
workers = create_workers(job_factory, options, &blk)
|
381
|
-
results = []
|
382
|
-
results_mutex = Mutex.new # arrays are not thread-safe
|
383
|
-
exception = nil
|
384
|
-
|
385
|
-
UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
|
386
|
-
in_threads(options) do |i|
|
387
|
-
worker = workers[i]
|
388
|
-
worker.thread = Thread.current
|
389
|
-
worked = false
|
390
|
-
|
391
|
-
begin
|
392
|
-
loop do
|
393
|
-
break if exception
|
394
|
-
item, index = job_factory.next
|
395
|
-
break unless index
|
396
|
-
|
397
|
-
if options[:isolation]
|
398
|
-
worker = replace_worker(job_factory, workers, i, options, blk) if worked
|
399
|
-
worked = true
|
400
|
-
worker.thread = Thread.current
|
401
|
-
end
|
402
|
-
|
403
|
-
begin
|
404
|
-
result = with_instrumentation item, index, options do
|
405
|
-
worker.work(job_factory.pack(item, index))
|
406
|
-
end
|
407
|
-
results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
|
408
|
-
rescue StandardError => e
|
409
|
-
exception = e
|
410
|
-
if Kill === exception
|
411
|
-
(workers - [worker]).each do |w|
|
412
|
-
w.thread.kill if w.thread
|
413
|
-
UserInterruptHandler.kill(w.pid)
|
414
|
-
end
|
415
|
-
end
|
416
|
-
end
|
417
|
-
end
|
418
|
-
ensure
|
419
|
-
worker.stop
|
420
|
-
end
|
421
|
-
end
|
422
|
-
end
|
423
|
-
exception || results
|
424
|
-
end
|
425
|
-
|
426
|
-
def replace_worker(job_factory, workers, i, options, blk)
|
427
|
-
options[:mutex].synchronize do
|
428
|
-
# old worker is no longer used ... stop it
|
429
|
-
worker = workers[i]
|
430
|
-
worker.stop
|
431
|
-
|
432
|
-
# create a new replacement worker
|
433
|
-
running = workers - [worker]
|
434
|
-
workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk)
|
435
|
-
end
|
436
|
-
end
|
437
|
-
|
438
|
-
def create_workers(job_factory, options, &block)
|
439
|
-
workers = []
|
440
|
-
Array.new(options[:count]).each_with_index do |_, i|
|
441
|
-
workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block)
|
442
|
-
end
|
443
|
-
workers
|
444
|
-
end
|
445
|
-
|
446
|
-
def worker(job_factory, options, &block)
|
447
|
-
child_read, parent_write = IO.pipe
|
448
|
-
parent_read, child_write = IO.pipe
|
449
|
-
|
450
|
-
pid = Process.fork do
|
451
|
-
self.worker_number = options[:worker_number]
|
452
|
-
|
453
|
-
begin
|
454
|
-
options.delete(:started_workers).each(&:close_pipes)
|
455
|
-
|
456
|
-
parent_write.close
|
457
|
-
parent_read.close
|
458
|
-
|
459
|
-
process_incoming_jobs(child_read, child_write, job_factory, options, &block)
|
460
|
-
ensure
|
461
|
-
child_read.close
|
462
|
-
child_write.close
|
463
|
-
end
|
464
|
-
end
|
465
|
-
|
466
|
-
child_read.close
|
467
|
-
child_write.close
|
468
|
-
|
469
|
-
Worker.new(parent_read, parent_write, pid)
|
470
|
-
end
|
471
|
-
|
472
|
-
def process_incoming_jobs(read, write, job_factory, options, &block)
|
473
|
-
until read.eof?
|
474
|
-
data = Marshal.load(read)
|
475
|
-
item, index = job_factory.unpack(data)
|
476
|
-
result = begin
|
477
|
-
call_with_index(item, index, options, &block)
|
478
|
-
# https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
|
479
|
-
rescue NoMemoryError, SignalException, Interrupt, SystemExit
|
480
|
-
raise $!
|
481
|
-
rescue Exception
|
482
|
-
ExceptionWrapper.new($!)
|
483
|
-
end
|
484
|
-
begin
|
485
|
-
Marshal.dump(result, write)
|
486
|
-
rescue Errno::EPIPE
|
487
|
-
return # parent thread already dead
|
488
|
-
end
|
489
|
-
end
|
490
|
-
end
|
491
|
-
|
492
|
-
# options is either a Integer or a Hash with :count
|
493
|
-
def extract_count_from_options(options)
|
494
|
-
if options.is_a?(Hash)
|
495
|
-
count = options[:count]
|
496
|
-
else
|
497
|
-
count = options
|
498
|
-
options = {}
|
499
|
-
end
|
500
|
-
[count, options]
|
501
|
-
end
|
502
|
-
|
503
|
-
def call_with_index(item, index, options, &block)
|
504
|
-
args = [item]
|
505
|
-
args << index if options[:with_index]
|
506
|
-
if options[:return_results]
|
507
|
-
block.call(*args)
|
508
|
-
else
|
509
|
-
block.call(*args)
|
510
|
-
nil # avoid GC overhead of passing large results around
|
511
|
-
end
|
512
|
-
end
|
513
|
-
|
514
|
-
def with_instrumentation(item, index, options)
|
515
|
-
on_start = options[:start]
|
516
|
-
on_finish = options[:finish]
|
517
|
-
options[:mutex].synchronize { on_start.call(item, index) } if on_start
|
518
|
-
result = yield
|
519
|
-
options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish
|
520
|
-
result unless options[:preserve_results] == false
|
521
|
-
end
|
522
|
-
end
|
523
|
-
end
|