brakeman 5.0.2 → 5.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGES.md +6 -0
- data/bundle/load.rb +0 -1
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby20_parser.rb +278 -273
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby20_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby21_parser.rb +291 -286
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby21_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby22_parser.rb +297 -292
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby22_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby23_parser.rb +295 -290
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby23_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby24_parser.rb +296 -291
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby24_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby25_parser.rb +297 -292
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby25_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby26_parser.rb +301 -296
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby26_parser.y +3 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby27_parser.rb +2528 -2480
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby27_parser.y +26 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.rb +2528 -2480
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.y +26 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby_parser.yy +30 -0
- data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/tools/ripper.rb +1 -1
- data/lib/brakeman.rb +0 -4
- data/lib/brakeman/checks/check_detailed_exceptions.rb +1 -1
- data/lib/brakeman/checks/check_evaluation.rb +1 -1
- data/lib/brakeman/checks/check_sql.rb +2 -15
- data/lib/brakeman/checks/check_verb_confusion.rb +1 -1
- data/lib/brakeman/file_parser.rb +14 -36
- data/lib/brakeman/options.rb +1 -1
- data/lib/brakeman/processors/alias_processor.rb +7 -52
- data/lib/brakeman/processors/controller_alias_processor.rb +43 -6
- data/lib/brakeman/processors/lib/call_conversion_helper.rb +0 -10
- data/lib/brakeman/processors/library_processor.rb +0 -9
- data/lib/brakeman/report.rb +1 -4
- data/lib/brakeman/report/ignore/interactive.rb +1 -1
- data/lib/brakeman/scanner.rb +0 -3
- data/lib/brakeman/tracker.rb +4 -33
- data/lib/brakeman/tracker/collection.rb +5 -27
- data/lib/brakeman/util.rb +0 -8
- data/lib/brakeman/version.rb +1 -1
- metadata +2 -8
- data/bundle/ruby/2.7.0/gems/parallel-1.20.1/MIT-LICENSE.txt +0 -20
- data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel.rb +0 -523
- data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/processor_count.rb +0 -42
- data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/version.rb +0 -3
- data/lib/brakeman/report/report_github.rb +0 -31
- data/lib/brakeman/tracker/method_info.rb +0 -29
@@ -76,8 +76,6 @@ module Brakeman
|
|
76
76
|
|
77
77
|
#Have to do this because first element is :array and we have to skip it
|
78
78
|
array[1..-1][index] or original_exp
|
79
|
-
elsif all_literals? array
|
80
|
-
safe_literal(array.line)
|
81
79
|
else
|
82
80
|
original_exp
|
83
81
|
end
|
@@ -94,13 +92,5 @@ module Brakeman
|
|
94
92
|
original_exp
|
95
93
|
end
|
96
94
|
end
|
97
|
-
|
98
|
-
def hash_values_at hash, keys
|
99
|
-
values = keys.map do |key|
|
100
|
-
process_hash_access hash, key
|
101
|
-
end
|
102
|
-
|
103
|
-
Sexp.new(:array).concat(values).line(hash.line)
|
104
|
-
end
|
105
95
|
end
|
106
96
|
end
|
@@ -54,15 +54,6 @@ class Brakeman::LibraryProcessor < Brakeman::BaseProcessor
|
|
54
54
|
|
55
55
|
def process_call exp
|
56
56
|
if process_call_defn? exp
|
57
|
-
exp
|
58
|
-
elsif @current_method.nil? and exp.target.nil? and (@current_class or @current_module)
|
59
|
-
# Methods called inside class / module
|
60
|
-
case exp.method
|
61
|
-
when :include
|
62
|
-
module_name = class_name(exp.first_arg)
|
63
|
-
(@current_class || @current_module).add_include module_name
|
64
|
-
end
|
65
|
-
|
66
57
|
exp
|
67
58
|
else
|
68
59
|
process_default exp
|
data/lib/brakeman/report.rb
CHANGED
@@ -6,7 +6,7 @@ require 'brakeman/report/report_base'
|
|
6
6
|
class Brakeman::Report
|
7
7
|
attr_reader :tracker
|
8
8
|
|
9
|
-
VALID_FORMATS = [:to_html, :to_pdf, :to_csv, :to_json, :to_tabs, :to_hash, :to_s, :to_markdown, :to_codeclimate, :to_plain, :to_text, :to_junit
|
9
|
+
VALID_FORMATS = [:to_html, :to_pdf, :to_csv, :to_json, :to_tabs, :to_hash, :to_s, :to_markdown, :to_codeclimate, :to_plain, :to_text, :to_junit]
|
10
10
|
|
11
11
|
def initialize tracker
|
12
12
|
@app_tree = tracker.app_tree
|
@@ -48,9 +48,6 @@ class Brakeman::Report
|
|
48
48
|
when :to_sonar
|
49
49
|
require_report 'sonar'
|
50
50
|
Brakeman::Report::Sonar
|
51
|
-
when :to_github
|
52
|
-
require_report 'github'
|
53
|
-
Brakeman::Report::Github
|
54
51
|
else
|
55
52
|
raise "Invalid format: #{format}. Should be one of #{VALID_FORMATS.inspect}"
|
56
53
|
end
|
data/lib/brakeman/scanner.rb
CHANGED
data/lib/brakeman/tracker.rb
CHANGED
@@ -35,7 +35,6 @@ class Brakeman::Tracker
|
|
35
35
|
#class they are.
|
36
36
|
@models = {}
|
37
37
|
@models[UNKNOWN_MODEL] = Brakeman::Model.new(UNKNOWN_MODEL, nil, @app_tree.file_path("NOT_REAL.rb"), nil, self)
|
38
|
-
@method_cache = {}
|
39
38
|
@routes = {}
|
40
39
|
@initializers = {}
|
41
40
|
@errors = []
|
@@ -100,8 +99,8 @@ class Brakeman::Tracker
|
|
100
99
|
classes.each do |set|
|
101
100
|
set.each do |set_name, collection|
|
102
101
|
collection.each_method do |method_name, definition|
|
103
|
-
src = definition
|
104
|
-
yield src, set_name, method_name, definition
|
102
|
+
src = definition[:src]
|
103
|
+
yield src, set_name, method_name, definition[:file]
|
105
104
|
end
|
106
105
|
end
|
107
106
|
end
|
@@ -221,34 +220,6 @@ class Brakeman::Tracker
|
|
221
220
|
nil
|
222
221
|
end
|
223
222
|
|
224
|
-
def find_method method_name, class_name, method_type = :instance
|
225
|
-
return nil unless method_name.is_a? Symbol
|
226
|
-
|
227
|
-
klass = find_class(class_name)
|
228
|
-
return nil unless klass
|
229
|
-
|
230
|
-
cache_key = [klass, method_name, method_type]
|
231
|
-
|
232
|
-
if method = @method_cache[cache_key]
|
233
|
-
return method
|
234
|
-
end
|
235
|
-
|
236
|
-
if method = klass.get_method(method_name, method_type)
|
237
|
-
return method
|
238
|
-
else
|
239
|
-
# Check modules included for method definition
|
240
|
-
# TODO: only for instance methods, otherwise check extends!
|
241
|
-
klass.includes.each do |included_name|
|
242
|
-
if method = find_method(method_name, included_name, method_type)
|
243
|
-
return (@method_cache[cache_key] = method)
|
244
|
-
end
|
245
|
-
end
|
246
|
-
|
247
|
-
# Not in any included modules, check the parent
|
248
|
-
@method_cache[cache_key] = find_method(method_name, klass.parent)
|
249
|
-
end
|
250
|
-
end
|
251
|
-
|
252
223
|
def index_call_sites
|
253
224
|
finder = Brakeman::FindAllCalls.new self
|
254
225
|
|
@@ -314,8 +285,8 @@ class Brakeman::Tracker
|
|
314
285
|
method_sets.each do |set|
|
315
286
|
set.each do |set_name, info|
|
316
287
|
info.each_method do |method_name, definition|
|
317
|
-
src = definition
|
318
|
-
finder.process_source src, :class => set_name, :method => method_name, :file => definition
|
288
|
+
src = definition[:src]
|
289
|
+
finder.process_source src, :class => set_name, :method => method_name, :file => definition[:file]
|
319
290
|
end
|
320
291
|
end
|
321
292
|
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'brakeman/util'
|
2
|
-
require 'brakeman/tracker/method_info'
|
3
2
|
|
4
3
|
module Brakeman
|
5
4
|
class Collection
|
@@ -14,7 +13,6 @@ module Brakeman
|
|
14
13
|
@src = {}
|
15
14
|
@includes = []
|
16
15
|
@methods = { :public => {}, :private => {}, :protected => {} }
|
17
|
-
@class_methods = {}
|
18
16
|
@options = {}
|
19
17
|
@tracker = tracker
|
20
18
|
|
@@ -48,16 +46,11 @@ module Brakeman
|
|
48
46
|
end
|
49
47
|
|
50
48
|
def add_method visibility, name, src, file_name
|
51
|
-
meth_info = Brakeman::MethodInfo.new(name, src, self, file_name)
|
52
|
-
|
53
49
|
if src.node_type == :defs
|
54
|
-
@class_methods[name] = meth_info
|
55
|
-
|
56
|
-
# TODO fix this weirdness
|
57
50
|
name = :"#{src[1]}.#{name}"
|
58
51
|
end
|
59
52
|
|
60
|
-
@methods[visibility][name] =
|
53
|
+
@methods[visibility][name] = { :src => src, :file => file_name }
|
61
54
|
end
|
62
55
|
|
63
56
|
def each_method
|
@@ -68,31 +61,16 @@ module Brakeman
|
|
68
61
|
end
|
69
62
|
end
|
70
63
|
|
71
|
-
def get_method name
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
when :instance
|
76
|
-
get_instance_method name
|
77
|
-
else
|
78
|
-
raise "Unexpected method type: #{type.inspect}"
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
def get_instance_method name
|
83
|
-
@methods.each do |_vis, meths|
|
84
|
-
if meths[name]
|
85
|
-
return meths[name]
|
64
|
+
def get_method name
|
65
|
+
each_method do |n, info|
|
66
|
+
if n == name
|
67
|
+
return info
|
86
68
|
end
|
87
69
|
end
|
88
70
|
|
89
71
|
nil
|
90
72
|
end
|
91
73
|
|
92
|
-
def get_class_method name
|
93
|
-
@class_methods[name]
|
94
|
-
end
|
95
|
-
|
96
74
|
def file
|
97
75
|
@files.first
|
98
76
|
end
|
data/lib/brakeman/util.rb
CHANGED
@@ -142,14 +142,6 @@ module Brakeman::Util
|
|
142
142
|
nil
|
143
143
|
end
|
144
144
|
|
145
|
-
def hash_values hash
|
146
|
-
values = hash.each_sexp.each_slice(2).map do |_, value|
|
147
|
-
value
|
148
|
-
end
|
149
|
-
|
150
|
-
Sexp.new(:array).concat(values).line(hash.line)
|
151
|
-
end
|
152
|
-
|
153
145
|
#These are never modified
|
154
146
|
PARAMS_SEXP = Sexp.new(:params)
|
155
147
|
SESSION_SEXP = Sexp.new(:session)
|
data/lib/brakeman/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: brakeman
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.0.
|
4
|
+
version: 5.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Collins
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-06-
|
11
|
+
date: 2021-06-08 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Brakeman detects security vulnerabilities in Ruby on Rails applications
|
14
14
|
via static analysis.
|
@@ -132,10 +132,6 @@ files:
|
|
132
132
|
- bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/terminal/unix_stty.rb
|
133
133
|
- bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/version.rb
|
134
134
|
- bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/wrapper.rb
|
135
|
-
- bundle/ruby/2.7.0/gems/parallel-1.20.1/MIT-LICENSE.txt
|
136
|
-
- bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel.rb
|
137
|
-
- bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/processor_count.rb
|
138
|
-
- bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/version.rb
|
139
135
|
- bundle/ruby/2.7.0/gems/rexml-3.2.5/LICENSE.txt
|
140
136
|
- bundle/ruby/2.7.0/gems/rexml-3.2.5/NEWS.md
|
141
137
|
- bundle/ruby/2.7.0/gems/rexml-3.2.5/README.md
|
@@ -572,7 +568,6 @@ files:
|
|
572
568
|
- lib/brakeman/report/report_base.rb
|
573
569
|
- lib/brakeman/report/report_codeclimate.rb
|
574
570
|
- lib/brakeman/report/report_csv.rb
|
575
|
-
- lib/brakeman/report/report_github.rb
|
576
571
|
- lib/brakeman/report/report_hash.rb
|
577
572
|
- lib/brakeman/report/report_html.rb
|
578
573
|
- lib/brakeman/report/report_json.rb
|
@@ -602,7 +597,6 @@ files:
|
|
602
597
|
- lib/brakeman/tracker/constants.rb
|
603
598
|
- lib/brakeman/tracker/controller.rb
|
604
599
|
- lib/brakeman/tracker/library.rb
|
605
|
-
- lib/brakeman/tracker/method_info.rb
|
606
600
|
- lib/brakeman/tracker/model.rb
|
607
601
|
- lib/brakeman/tracker/template.rb
|
608
602
|
- lib/brakeman/util.rb
|
@@ -1,20 +0,0 @@
|
|
1
|
-
Copyright (C) 2013 Michael Grosser <michael@grosser.it>
|
2
|
-
|
3
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
-
a copy of this software and associated documentation files (the
|
5
|
-
"Software"), to deal in the Software without restriction, including
|
6
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
-
permit persons to whom the Software is furnished to do so, subject to
|
9
|
-
the following conditions:
|
10
|
-
|
11
|
-
The above copyright notice and this permission notice shall be
|
12
|
-
included in all copies or substantial portions of the Software.
|
13
|
-
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
@@ -1,523 +0,0 @@
|
|
1
|
-
require 'rbconfig'
|
2
|
-
require 'parallel/version'
|
3
|
-
require 'parallel/processor_count'
|
4
|
-
|
5
|
-
module Parallel
|
6
|
-
extend ProcessorCount
|
7
|
-
|
8
|
-
Stop = Object.new.freeze
|
9
|
-
|
10
|
-
class DeadWorker < StandardError
|
11
|
-
end
|
12
|
-
|
13
|
-
class Break < StandardError
|
14
|
-
attr_reader :value
|
15
|
-
def initialize(value = nil)
|
16
|
-
@value = value
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
class Kill < Break
|
21
|
-
end
|
22
|
-
|
23
|
-
class UndumpableException < StandardError
|
24
|
-
attr_reader :backtrace
|
25
|
-
def initialize(original)
|
26
|
-
super "#{original.class}: #{original.message}"
|
27
|
-
@backtrace = original.backtrace
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
class ExceptionWrapper
|
32
|
-
attr_reader :exception
|
33
|
-
def initialize(exception)
|
34
|
-
# Remove the bindings stack added by the better_errors gem,
|
35
|
-
# because it cannot be marshalled
|
36
|
-
if exception.instance_variable_defined? :@__better_errors_bindings_stack
|
37
|
-
exception.send :remove_instance_variable, :@__better_errors_bindings_stack
|
38
|
-
end
|
39
|
-
|
40
|
-
@exception =
|
41
|
-
begin
|
42
|
-
Marshal.dump(exception) && exception
|
43
|
-
rescue
|
44
|
-
UndumpableException.new(exception)
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
class Worker
|
50
|
-
attr_reader :pid, :read, :write
|
51
|
-
attr_accessor :thread
|
52
|
-
def initialize(read, write, pid)
|
53
|
-
@read, @write, @pid = read, write, pid
|
54
|
-
end
|
55
|
-
|
56
|
-
def stop
|
57
|
-
close_pipes
|
58
|
-
wait # if it goes zombie, rather wait here to be able to debug
|
59
|
-
end
|
60
|
-
|
61
|
-
# might be passed to started_processes and simultaneously closed by another thread
|
62
|
-
# when running in isolation mode, so we have to check if it is closed before closing
|
63
|
-
def close_pipes
|
64
|
-
read.close unless read.closed?
|
65
|
-
write.close unless write.closed?
|
66
|
-
end
|
67
|
-
|
68
|
-
def work(data)
|
69
|
-
begin
|
70
|
-
Marshal.dump(data, write)
|
71
|
-
rescue Errno::EPIPE
|
72
|
-
raise DeadWorker
|
73
|
-
end
|
74
|
-
|
75
|
-
result = begin
|
76
|
-
Marshal.load(read)
|
77
|
-
rescue EOFError
|
78
|
-
raise DeadWorker
|
79
|
-
end
|
80
|
-
raise result.exception if ExceptionWrapper === result
|
81
|
-
result
|
82
|
-
end
|
83
|
-
|
84
|
-
private
|
85
|
-
|
86
|
-
def wait
|
87
|
-
Process.wait(pid)
|
88
|
-
rescue Interrupt
|
89
|
-
# process died
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
class JobFactory
|
94
|
-
def initialize(source, mutex)
|
95
|
-
@lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
|
96
|
-
@source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
|
97
|
-
@mutex = mutex
|
98
|
-
@index = -1
|
99
|
-
@stopped = false
|
100
|
-
end
|
101
|
-
|
102
|
-
def next
|
103
|
-
if producer?
|
104
|
-
# - index and item stay in sync
|
105
|
-
# - do not call lambda after it has returned Stop
|
106
|
-
item, index = @mutex.synchronize do
|
107
|
-
return if @stopped
|
108
|
-
item = @lambda.call
|
109
|
-
@stopped = (item == Stop)
|
110
|
-
return if @stopped
|
111
|
-
[item, @index += 1]
|
112
|
-
end
|
113
|
-
else
|
114
|
-
index = @mutex.synchronize { @index += 1 }
|
115
|
-
return if index >= size
|
116
|
-
item = @source[index]
|
117
|
-
end
|
118
|
-
[item, index]
|
119
|
-
end
|
120
|
-
|
121
|
-
def size
|
122
|
-
if producer?
|
123
|
-
Float::INFINITY
|
124
|
-
else
|
125
|
-
@source.size
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
|
-
# generate item that is sent to workers
|
130
|
-
# just index is faster + less likely to blow up with unserializable errors
|
131
|
-
def pack(item, index)
|
132
|
-
producer? ? [item, index] : index
|
133
|
-
end
|
134
|
-
|
135
|
-
# unpack item that is sent to workers
|
136
|
-
def unpack(data)
|
137
|
-
producer? ? data : [@source[data], data]
|
138
|
-
end
|
139
|
-
|
140
|
-
private
|
141
|
-
|
142
|
-
def producer?
|
143
|
-
@lambda
|
144
|
-
end
|
145
|
-
|
146
|
-
def queue_wrapper(array)
|
147
|
-
array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
|
148
|
-
end
|
149
|
-
end
|
150
|
-
|
151
|
-
class UserInterruptHandler
|
152
|
-
INTERRUPT_SIGNAL = :SIGINT
|
153
|
-
|
154
|
-
class << self
|
155
|
-
# kill all these pids or threads if user presses Ctrl+c
|
156
|
-
def kill_on_ctrl_c(pids, options)
|
157
|
-
@to_be_killed ||= []
|
158
|
-
old_interrupt = nil
|
159
|
-
signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
|
160
|
-
|
161
|
-
if @to_be_killed.empty?
|
162
|
-
old_interrupt = trap_interrupt(signal) do
|
163
|
-
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
164
|
-
@to_be_killed.flatten.each { |pid| kill(pid) }
|
165
|
-
end
|
166
|
-
end
|
167
|
-
|
168
|
-
@to_be_killed << pids
|
169
|
-
|
170
|
-
yield
|
171
|
-
ensure
|
172
|
-
@to_be_killed.pop # do not kill pids that could be used for new processes
|
173
|
-
restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
|
174
|
-
end
|
175
|
-
|
176
|
-
def kill(thing)
|
177
|
-
Process.kill(:KILL, thing)
|
178
|
-
rescue Errno::ESRCH
|
179
|
-
# some linux systems already automatically killed the children at this point
|
180
|
-
# so we just ignore them not being there
|
181
|
-
end
|
182
|
-
|
183
|
-
private
|
184
|
-
|
185
|
-
def trap_interrupt(signal)
|
186
|
-
old = Signal.trap signal, 'IGNORE'
|
187
|
-
|
188
|
-
Signal.trap signal do
|
189
|
-
yield
|
190
|
-
if !old || old == "DEFAULT"
|
191
|
-
raise Interrupt
|
192
|
-
else
|
193
|
-
old.call
|
194
|
-
end
|
195
|
-
end
|
196
|
-
|
197
|
-
old
|
198
|
-
end
|
199
|
-
|
200
|
-
def restore_interrupt(old, signal)
|
201
|
-
Signal.trap signal, old
|
202
|
-
end
|
203
|
-
end
|
204
|
-
end
|
205
|
-
|
206
|
-
class << self
|
207
|
-
def in_threads(options={:count => 2})
|
208
|
-
threads = []
|
209
|
-
count, _ = extract_count_from_options(options)
|
210
|
-
|
211
|
-
Thread.handle_interrupt(Exception => :never) do
|
212
|
-
begin
|
213
|
-
Thread.handle_interrupt(Exception => :immediate) do
|
214
|
-
count.times do |i|
|
215
|
-
threads << Thread.new { yield(i) }
|
216
|
-
end
|
217
|
-
threads.map(&:value)
|
218
|
-
end
|
219
|
-
ensure
|
220
|
-
threads.each(&:kill)
|
221
|
-
end
|
222
|
-
end
|
223
|
-
end
|
224
|
-
|
225
|
-
def in_processes(options = {}, &block)
|
226
|
-
count, options = extract_count_from_options(options)
|
227
|
-
count ||= processor_count
|
228
|
-
map(0...count, options.merge(:in_processes => count), &block)
|
229
|
-
end
|
230
|
-
|
231
|
-
def each(array, options={}, &block)
|
232
|
-
map(array, options.merge(:preserve_results => false), &block)
|
233
|
-
end
|
234
|
-
|
235
|
-
def any?(*args, &block)
|
236
|
-
raise "You must provide a block when calling #any?" if block.nil?
|
237
|
-
!each(*args) { |*a| raise Kill if block.call(*a) }
|
238
|
-
end
|
239
|
-
|
240
|
-
def all?(*args, &block)
|
241
|
-
raise "You must provide a block when calling #all?" if block.nil?
|
242
|
-
!!each(*args) { |*a| raise Kill unless block.call(*a) }
|
243
|
-
end
|
244
|
-
|
245
|
-
def each_with_index(array, options={}, &block)
|
246
|
-
each(array, options.merge(:with_index => true), &block)
|
247
|
-
end
|
248
|
-
|
249
|
-
def map(source, options = {}, &block)
|
250
|
-
options = options.dup
|
251
|
-
options[:mutex] = Mutex.new
|
252
|
-
|
253
|
-
if options[:in_processes] && options[:in_threads]
|
254
|
-
raise ArgumentError.new("Please specify only one of `in_processes` or `in_threads`.")
|
255
|
-
elsif RUBY_PLATFORM =~ /java/ and not options[:in_processes]
|
256
|
-
method = :in_threads
|
257
|
-
size = options[method] || processor_count
|
258
|
-
elsif options[:in_threads]
|
259
|
-
method = :in_threads
|
260
|
-
size = options[method]
|
261
|
-
else
|
262
|
-
method = :in_processes
|
263
|
-
if Process.respond_to?(:fork)
|
264
|
-
size = options[method] || processor_count
|
265
|
-
else
|
266
|
-
warn "Process.fork is not supported by this Ruby"
|
267
|
-
size = 0
|
268
|
-
end
|
269
|
-
end
|
270
|
-
|
271
|
-
job_factory = JobFactory.new(source, options[:mutex])
|
272
|
-
size = [job_factory.size, size].min
|
273
|
-
|
274
|
-
options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
|
275
|
-
add_progress_bar!(job_factory, options)
|
276
|
-
|
277
|
-
result =
|
278
|
-
if size == 0
|
279
|
-
work_direct(job_factory, options, &block)
|
280
|
-
elsif method == :in_threads
|
281
|
-
work_in_threads(job_factory, options.merge(:count => size), &block)
|
282
|
-
else
|
283
|
-
work_in_processes(job_factory, options.merge(:count => size), &block)
|
284
|
-
end
|
285
|
-
|
286
|
-
return result.value if result.is_a?(Break)
|
287
|
-
raise result if result.is_a?(Exception)
|
288
|
-
options[:return_results] ? result : source
|
289
|
-
end
|
290
|
-
|
291
|
-
def map_with_index(array, options={}, &block)
|
292
|
-
map(array, options.merge(:with_index => true), &block)
|
293
|
-
end
|
294
|
-
|
295
|
-
def flat_map(*args, &block)
|
296
|
-
map(*args, &block).flatten(1)
|
297
|
-
end
|
298
|
-
|
299
|
-
def worker_number
|
300
|
-
Thread.current[:parallel_worker_number]
|
301
|
-
end
|
302
|
-
|
303
|
-
# TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
|
304
|
-
def worker_number=(worker_num)
|
305
|
-
Thread.current[:parallel_worker_number] = worker_num
|
306
|
-
end
|
307
|
-
|
308
|
-
private
|
309
|
-
|
310
|
-
def add_progress_bar!(job_factory, options)
|
311
|
-
if progress_options = options[:progress]
|
312
|
-
raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
|
313
|
-
require 'ruby-progressbar'
|
314
|
-
|
315
|
-
if progress_options == true
|
316
|
-
progress_options = { title: "Progress" }
|
317
|
-
elsif progress_options.respond_to? :to_str
|
318
|
-
progress_options = { title: progress_options.to_str }
|
319
|
-
end
|
320
|
-
|
321
|
-
progress_options = {
|
322
|
-
total: job_factory.size,
|
323
|
-
format: '%t |%E | %B | %a'
|
324
|
-
}.merge(progress_options)
|
325
|
-
|
326
|
-
progress = ProgressBar.create(progress_options)
|
327
|
-
old_finish = options[:finish]
|
328
|
-
options[:finish] = lambda do |item, i, result|
|
329
|
-
old_finish.call(item, i, result) if old_finish
|
330
|
-
progress.increment
|
331
|
-
end
|
332
|
-
end
|
333
|
-
end
|
334
|
-
|
335
|
-
def work_direct(job_factory, options, &block)
|
336
|
-
self.worker_number = 0
|
337
|
-
results = []
|
338
|
-
exception = nil
|
339
|
-
begin
|
340
|
-
while set = job_factory.next
|
341
|
-
item, index = set
|
342
|
-
results << with_instrumentation(item, index, options) do
|
343
|
-
call_with_index(item, index, options, &block)
|
344
|
-
end
|
345
|
-
end
|
346
|
-
rescue
|
347
|
-
exception = $!
|
348
|
-
end
|
349
|
-
exception || results
|
350
|
-
ensure
|
351
|
-
self.worker_number = nil
|
352
|
-
end
|
353
|
-
|
354
|
-
def work_in_threads(job_factory, options, &block)
|
355
|
-
raise "interrupt_signal is no longer supported for threads" if options[:interrupt_signal]
|
356
|
-
results = []
|
357
|
-
results_mutex = Mutex.new # arrays are not thread-safe on jRuby
|
358
|
-
exception = nil
|
359
|
-
|
360
|
-
in_threads(options) do |worker_num|
|
361
|
-
self.worker_number = worker_num
|
362
|
-
# as long as there are more jobs, work on one of them
|
363
|
-
while !exception && set = job_factory.next
|
364
|
-
begin
|
365
|
-
item, index = set
|
366
|
-
result = with_instrumentation item, index, options do
|
367
|
-
call_with_index(item, index, options, &block)
|
368
|
-
end
|
369
|
-
results_mutex.synchronize { results[index] = result }
|
370
|
-
rescue
|
371
|
-
exception = $!
|
372
|
-
end
|
373
|
-
end
|
374
|
-
end
|
375
|
-
|
376
|
-
exception || results
|
377
|
-
end
|
378
|
-
|
379
|
-
def work_in_processes(job_factory, options, &blk)
|
380
|
-
workers = create_workers(job_factory, options, &blk)
|
381
|
-
results = []
|
382
|
-
results_mutex = Mutex.new # arrays are not thread-safe
|
383
|
-
exception = nil
|
384
|
-
|
385
|
-
UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
|
386
|
-
in_threads(options) do |i|
|
387
|
-
worker = workers[i]
|
388
|
-
worker.thread = Thread.current
|
389
|
-
worked = false
|
390
|
-
|
391
|
-
begin
|
392
|
-
loop do
|
393
|
-
break if exception
|
394
|
-
item, index = job_factory.next
|
395
|
-
break unless index
|
396
|
-
|
397
|
-
if options[:isolation]
|
398
|
-
worker = replace_worker(job_factory, workers, i, options, blk) if worked
|
399
|
-
worked = true
|
400
|
-
worker.thread = Thread.current
|
401
|
-
end
|
402
|
-
|
403
|
-
begin
|
404
|
-
result = with_instrumentation item, index, options do
|
405
|
-
worker.work(job_factory.pack(item, index))
|
406
|
-
end
|
407
|
-
results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
|
408
|
-
rescue StandardError => e
|
409
|
-
exception = e
|
410
|
-
if Kill === exception
|
411
|
-
(workers - [worker]).each do |w|
|
412
|
-
w.thread.kill if w.thread
|
413
|
-
UserInterruptHandler.kill(w.pid)
|
414
|
-
end
|
415
|
-
end
|
416
|
-
end
|
417
|
-
end
|
418
|
-
ensure
|
419
|
-
worker.stop
|
420
|
-
end
|
421
|
-
end
|
422
|
-
end
|
423
|
-
exception || results
|
424
|
-
end
|
425
|
-
|
426
|
-
def replace_worker(job_factory, workers, i, options, blk)
|
427
|
-
options[:mutex].synchronize do
|
428
|
-
# old worker is no longer used ... stop it
|
429
|
-
worker = workers[i]
|
430
|
-
worker.stop
|
431
|
-
|
432
|
-
# create a new replacement worker
|
433
|
-
running = workers - [worker]
|
434
|
-
workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk)
|
435
|
-
end
|
436
|
-
end
|
437
|
-
|
438
|
-
def create_workers(job_factory, options, &block)
|
439
|
-
workers = []
|
440
|
-
Array.new(options[:count]).each_with_index do |_, i|
|
441
|
-
workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block)
|
442
|
-
end
|
443
|
-
workers
|
444
|
-
end
|
445
|
-
|
446
|
-
def worker(job_factory, options, &block)
|
447
|
-
child_read, parent_write = IO.pipe
|
448
|
-
parent_read, child_write = IO.pipe
|
449
|
-
|
450
|
-
pid = Process.fork do
|
451
|
-
self.worker_number = options[:worker_number]
|
452
|
-
|
453
|
-
begin
|
454
|
-
options.delete(:started_workers).each(&:close_pipes)
|
455
|
-
|
456
|
-
parent_write.close
|
457
|
-
parent_read.close
|
458
|
-
|
459
|
-
process_incoming_jobs(child_read, child_write, job_factory, options, &block)
|
460
|
-
ensure
|
461
|
-
child_read.close
|
462
|
-
child_write.close
|
463
|
-
end
|
464
|
-
end
|
465
|
-
|
466
|
-
child_read.close
|
467
|
-
child_write.close
|
468
|
-
|
469
|
-
Worker.new(parent_read, parent_write, pid)
|
470
|
-
end
|
471
|
-
|
472
|
-
def process_incoming_jobs(read, write, job_factory, options, &block)
|
473
|
-
until read.eof?
|
474
|
-
data = Marshal.load(read)
|
475
|
-
item, index = job_factory.unpack(data)
|
476
|
-
result = begin
|
477
|
-
call_with_index(item, index, options, &block)
|
478
|
-
# https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
|
479
|
-
rescue NoMemoryError, SignalException, Interrupt, SystemExit
|
480
|
-
raise $!
|
481
|
-
rescue Exception
|
482
|
-
ExceptionWrapper.new($!)
|
483
|
-
end
|
484
|
-
begin
|
485
|
-
Marshal.dump(result, write)
|
486
|
-
rescue Errno::EPIPE
|
487
|
-
return # parent thread already dead
|
488
|
-
end
|
489
|
-
end
|
490
|
-
end
|
491
|
-
|
492
|
-
# options is either a Integer or a Hash with :count
|
493
|
-
def extract_count_from_options(options)
|
494
|
-
if options.is_a?(Hash)
|
495
|
-
count = options[:count]
|
496
|
-
else
|
497
|
-
count = options
|
498
|
-
options = {}
|
499
|
-
end
|
500
|
-
[count, options]
|
501
|
-
end
|
502
|
-
|
503
|
-
def call_with_index(item, index, options, &block)
|
504
|
-
args = [item]
|
505
|
-
args << index if options[:with_index]
|
506
|
-
if options[:return_results]
|
507
|
-
block.call(*args)
|
508
|
-
else
|
509
|
-
block.call(*args)
|
510
|
-
nil # avoid GC overhead of passing large results around
|
511
|
-
end
|
512
|
-
end
|
513
|
-
|
514
|
-
def with_instrumentation(item, index, options)
|
515
|
-
on_start = options[:start]
|
516
|
-
on_finish = options[:finish]
|
517
|
-
options[:mutex].synchronize { on_start.call(item, index) } if on_start
|
518
|
-
result = yield
|
519
|
-
options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish
|
520
|
-
result unless options[:preserve_results] == false
|
521
|
-
end
|
522
|
-
end
|
523
|
-
end
|