buildkite-test_collector 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/CHANGELOG.md +19 -0
- data/CODE_OF_CONDUCT.md +133 -0
- data/Gemfile +10 -0
- data/Gemfile.lock +50 -0
- data/LICENSE.txt +21 -0
- data/README.md +103 -0
- data/Rakefile +8 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/buildkite-test_collector.gemspec +32 -0
- data/buildkite.yaml +8 -0
- data/lib/buildkite/test_collector/ci.rb +86 -0
- data/lib/buildkite/test_collector/http_client.rb +35 -0
- data/lib/buildkite/test_collector/library_hooks/minitest.rb +16 -0
- data/lib/buildkite/test_collector/library_hooks/rspec.rb +35 -0
- data/lib/buildkite/test_collector/logger.rb +19 -0
- data/lib/buildkite/test_collector/minitest_plugin/reporter.rb +34 -0
- data/lib/buildkite/test_collector/minitest_plugin/trace.rb +102 -0
- data/lib/buildkite/test_collector/minitest_plugin.rb +27 -0
- data/lib/buildkite/test_collector/network.rb +77 -0
- data/lib/buildkite/test_collector/object.rb +20 -0
- data/lib/buildkite/test_collector/rspec_plugin/reporter.rb +95 -0
- data/lib/buildkite/test_collector/rspec_plugin/trace.rb +87 -0
- data/lib/buildkite/test_collector/session.rb +331 -0
- data/lib/buildkite/test_collector/socket_connection.rb +157 -0
- data/lib/buildkite/test_collector/tracer.rb +65 -0
- data/lib/buildkite/test_collector/uploader.rb +73 -0
- data/lib/buildkite/test_collector/version.rb +8 -0
- data/lib/buildkite/test_collector.rb +84 -0
- data/lib/minitest/buildkite_collector_plugin.rb +7 -0
- metadata +139 -0
@@ -0,0 +1,102 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/core_ext/hash/indifferent_access"
|
4
|
+
|
5
|
+
module Buildkite::TestCollector::MinitestPlugin
|
6
|
+
class Trace
|
7
|
+
attr_accessor :example, :failure_reason, :failure_expanded
|
8
|
+
attr_reader :id, :history
|
9
|
+
|
10
|
+
RESULT_CODES = {
|
11
|
+
'.' => 'passed',
|
12
|
+
'F' => 'failed',
|
13
|
+
'E' => 'failed',
|
14
|
+
'S' => 'pending',
|
15
|
+
}
|
16
|
+
|
17
|
+
FILE_PATH_REGEX = /^(.*?\.(rb|feature))/
|
18
|
+
|
19
|
+
def initialize(example, history:, failure_reason: nil, failure_expanded: [])
|
20
|
+
@id = SecureRandom.uuid
|
21
|
+
@example = example
|
22
|
+
@history = history
|
23
|
+
@failure_reason = failure_reason
|
24
|
+
@failure_expanded = failure_expanded
|
25
|
+
end
|
26
|
+
|
27
|
+
def result
|
28
|
+
RESULT_CODES[example.result_code]
|
29
|
+
end
|
30
|
+
|
31
|
+
def source_location
|
32
|
+
@source_location ||= example.method(example.name).source_location
|
33
|
+
end
|
34
|
+
|
35
|
+
def as_hash
|
36
|
+
strip_invalid_utf8_chars(
|
37
|
+
id: id,
|
38
|
+
scope: example.class.name,
|
39
|
+
name: example.name,
|
40
|
+
identifier: identifier,
|
41
|
+
location: location,
|
42
|
+
file_name: file_name,
|
43
|
+
result: result,
|
44
|
+
failure_reason: failure_reason,
|
45
|
+
failure_expanded: failure_expanded,
|
46
|
+
history: history,
|
47
|
+
).with_indifferent_access.compact
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def location
|
53
|
+
if file_name
|
54
|
+
"#{file_name}:#{line_number}"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
alias_method :identifier, :location
|
58
|
+
|
59
|
+
def file_name
|
60
|
+
@file_name ||= File.join('./', source_location[0].delete_prefix(project_dir))
|
61
|
+
end
|
62
|
+
|
63
|
+
def line_number
|
64
|
+
@line_number ||= source_location[1]
|
65
|
+
end
|
66
|
+
|
67
|
+
def project_dir
|
68
|
+
if defined?(Rails) && Rails.respond_to?(:root)
|
69
|
+
Rails.root
|
70
|
+
else
|
71
|
+
Dir.getwd
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def failure_reason
|
76
|
+
@failure_reason ||= example.failure&.message
|
77
|
+
end
|
78
|
+
|
79
|
+
def failure_expanded
|
80
|
+
@failure_expanded ||= begin
|
81
|
+
example.failures.map do |failure|
|
82
|
+
{
|
83
|
+
expanded: failure.message,
|
84
|
+
backtrace: failure.backtrace,
|
85
|
+
}
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def strip_invalid_utf8_chars(object)
|
91
|
+
if object.is_a?(Hash)
|
92
|
+
Hash[object.map { |key, value| [key, strip_invalid_utf8_chars(value)] }]
|
93
|
+
elsif object.is_a?(Array)
|
94
|
+
object.map { |value| strip_invalid_utf8_chars(value) }
|
95
|
+
elsif object.is_a?(String)
|
96
|
+
object.encode('UTF-8', :invalid => :replace, :undef => :replace)
|
97
|
+
else
|
98
|
+
object
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "minitest_plugin/reporter"
|
4
|
+
require_relative "minitest_plugin/trace"
|
5
|
+
|
6
|
+
module Buildkite::TestCollector::MinitestPlugin
|
7
|
+
def before_setup
|
8
|
+
super
|
9
|
+
tracer = Buildkite::TestCollector::Tracer.new
|
10
|
+
# The _buildkite prefix here is added as a safeguard against name collisions
|
11
|
+
# as we are in the main thread
|
12
|
+
Thread.current[:_buildkite_tracer] = tracer
|
13
|
+
end
|
14
|
+
|
15
|
+
def before_teardown
|
16
|
+
super
|
17
|
+
|
18
|
+
tracer = Thread.current[:_buildkite_tracer]
|
19
|
+
if !tracer.nil?
|
20
|
+
Thread.current[:_buildkite_tracer] = nil
|
21
|
+
tracer.finalize
|
22
|
+
|
23
|
+
trace = Buildkite::TestCollector::MinitestPlugin::Trace.new(self, history: tracer.history)
|
24
|
+
Buildkite::TestCollector.uploader.traces[trace.source_location] = trace
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Buildkite::TestCollector
|
4
|
+
class Network
|
5
|
+
module NetHTTPPatch
|
6
|
+
def request(request, *args, &block)
|
7
|
+
unless uri = request.uri
|
8
|
+
protocol = use_ssl? ? "https" : "http"
|
9
|
+
uri = URI.join("#{protocol}://#{address}:#{port}", request.path)
|
10
|
+
end
|
11
|
+
|
12
|
+
detail = { method: request.method.upcase, url: uri.to_s, lib: "net-http" }
|
13
|
+
|
14
|
+
http_tracer = Buildkite::TestCollector::Uploader.tracer
|
15
|
+
http_tracer&.enter("http", **detail)
|
16
|
+
|
17
|
+
super
|
18
|
+
ensure
|
19
|
+
http_tracer&.leave
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
module VCRPatch
|
24
|
+
def handle
|
25
|
+
if request_type == :stubbed_by_vcr && tracer = Buildkite::TestCollector::Uploader.tracer
|
26
|
+
tracer.current_span.detail.merge!(stubbed: "vcr")
|
27
|
+
end
|
28
|
+
|
29
|
+
super
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
module HTTPPatch
|
34
|
+
def perform(request, options)
|
35
|
+
detail = { method: request.verb.to_s.upcase, url: request.uri.to_s, lib: "http" }
|
36
|
+
|
37
|
+
http_tracer = Buildkite::TestCollector::Uploader.tracer
|
38
|
+
http_tracer&.enter("http", **detail)
|
39
|
+
|
40
|
+
super
|
41
|
+
ensure
|
42
|
+
http_tracer&.leave
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
module WebMockPatch
|
47
|
+
def response_for_request(request_signature)
|
48
|
+
response_from_webmock = super
|
49
|
+
|
50
|
+
if response_from_webmock && tracer = Buildkite::TestCollector::Uploader.tracer
|
51
|
+
tracer.current_span.detail.merge!(stubbed: "webmock")
|
52
|
+
end
|
53
|
+
|
54
|
+
response_from_webmock
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.configure
|
59
|
+
if defined?(VCR)
|
60
|
+
require "vcr/request_handler"
|
61
|
+
VCR::RequestHandler.prepend(VCRPatch)
|
62
|
+
end
|
63
|
+
|
64
|
+
if defined?(WebMock)
|
65
|
+
WebMock::StubRegistry.prepend(WebMockPatch)
|
66
|
+
end
|
67
|
+
|
68
|
+
if defined?(Net) && defined?(Net::HTTP)
|
69
|
+
Net::HTTP.prepend(NetHTTPPatch)
|
70
|
+
end
|
71
|
+
|
72
|
+
if defined?(HTTP) && defined?(HTTP::Client)
|
73
|
+
HTTP::Client.prepend(HTTPPatch)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Buildkite::TestCollector
|
4
|
+
class Object
|
5
|
+
module CustomObjectSleep
|
6
|
+
def sleep(duration)
|
7
|
+
tracer = Buildkite::TestCollector::Uploader.tracer
|
8
|
+
tracer&.enter("sleep")
|
9
|
+
|
10
|
+
super
|
11
|
+
ensure
|
12
|
+
tracer&.leave
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.configure
|
17
|
+
::Object.prepend(CustomObjectSleep)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "time"
|
4
|
+
|
5
|
+
module Buildkite::TestCollector::RSpecPlugin
|
6
|
+
class Reporter
|
7
|
+
RSpec::Core::Formatters.register self, :example_passed, :example_failed, :example_pending, :dump_summary
|
8
|
+
|
9
|
+
attr_reader :output
|
10
|
+
|
11
|
+
def initialize(output)
|
12
|
+
@output = output
|
13
|
+
end
|
14
|
+
|
15
|
+
def handle_example(notification)
|
16
|
+
example = notification.example
|
17
|
+
trace = Buildkite::TestCollector.uploader.traces[example.id]
|
18
|
+
|
19
|
+
if trace
|
20
|
+
trace.example = example
|
21
|
+
if example.execution_result.status == :failed
|
22
|
+
trace.failure_reason, trace.failure_expanded = failure_info(notification)
|
23
|
+
end
|
24
|
+
Buildkite::TestCollector.session&.write_result(trace)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def dump_summary(notification)
|
29
|
+
if Buildkite::TestCollector.session.present?
|
30
|
+
examples_count = {
|
31
|
+
examples: notification.examples.count,
|
32
|
+
failed: notification.failed_examples.count,
|
33
|
+
pending: notification.pending_examples.count,
|
34
|
+
errors_outside_examples: notification.errors_outside_of_examples_count
|
35
|
+
}
|
36
|
+
|
37
|
+
Buildkite::TestCollector.session.close(examples_count)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
alias_method :example_passed, :handle_example
|
42
|
+
alias_method :example_failed, :handle_example
|
43
|
+
alias_method :example_pending, :handle_example
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
MULTIPLE_ERRORS = [
|
48
|
+
RSpec::Expectations::MultipleExpectationsNotMetError,
|
49
|
+
RSpec::Core::MultipleExceptionError
|
50
|
+
]
|
51
|
+
|
52
|
+
def failure_info(notification)
|
53
|
+
failure_expanded = []
|
54
|
+
|
55
|
+
if Buildkite::TestCollector::RSpecPlugin::Reporter::MULTIPLE_ERRORS.include?(notification.exception.class)
|
56
|
+
failure_reason = notification.exception.summary
|
57
|
+
notification.exception.all_exceptions.each do |exception|
|
58
|
+
# an example with multiple failures doesn't give us a
|
59
|
+
# separate message lines and backtrace object to send, so
|
60
|
+
# I've reached into RSpec internals and duplicated the
|
61
|
+
# construction of these
|
62
|
+
message_lines = RSpec::Core::Formatters::ExceptionPresenter.new(exception, notification.example).colorized_message_lines
|
63
|
+
|
64
|
+
failure_expanded << {
|
65
|
+
expanded: format_message_lines(message_lines),
|
66
|
+
backtrace: RSpec.configuration.backtrace_formatter.format_backtrace(exception.backtrace)
|
67
|
+
}
|
68
|
+
end
|
69
|
+
else
|
70
|
+
message_lines = notification.colorized_message_lines
|
71
|
+
failure_reason = strip_diff_colors(message_lines.shift)
|
72
|
+
|
73
|
+
failure_expanded << {
|
74
|
+
expanded: format_message_lines(message_lines),
|
75
|
+
backtrace: notification.formatted_backtrace
|
76
|
+
}
|
77
|
+
end
|
78
|
+
|
79
|
+
return failure_reason, failure_expanded
|
80
|
+
end
|
81
|
+
|
82
|
+
def format_message_lines(message_lines)
|
83
|
+
message_lines.map! { |l| strip_diff_colors(l) }
|
84
|
+
# the first line is sometimes blank, depending on the error reported
|
85
|
+
message_lines.shift if message_lines.first.blank?
|
86
|
+
# the last line is sometimes blank, depending on the error reported
|
87
|
+
message_lines.pop if message_lines.last.blank?
|
88
|
+
message_lines
|
89
|
+
end
|
90
|
+
|
91
|
+
def strip_diff_colors(string)
|
92
|
+
string.gsub(/\e\[([;\d]+)?m/, '')
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/core_ext/hash/indifferent_access"
|
4
|
+
|
5
|
+
module Buildkite::TestCollector::RSpecPlugin
|
6
|
+
class Trace
|
7
|
+
attr_accessor :example, :failure_reason, :failure_expanded
|
8
|
+
attr_reader :id, :history
|
9
|
+
|
10
|
+
FILE_PATH_REGEX = /^(.*?\.(rb|feature))/
|
11
|
+
|
12
|
+
def initialize(example, history:, failure_reason: nil, failure_expanded: [])
|
13
|
+
@id = SecureRandom.uuid
|
14
|
+
@example = example
|
15
|
+
@history = history
|
16
|
+
@failure_reason = failure_reason
|
17
|
+
@failure_expanded = failure_expanded
|
18
|
+
end
|
19
|
+
|
20
|
+
def result
|
21
|
+
case example.execution_result.status
|
22
|
+
when :passed; "passed"
|
23
|
+
when :failed; "failed"
|
24
|
+
when :pending; "skipped"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def as_hash
|
29
|
+
strip_invalid_utf8_chars(
|
30
|
+
id: id,
|
31
|
+
scope: example.example_group.metadata[:full_description],
|
32
|
+
name: example.description,
|
33
|
+
identifier: example.id,
|
34
|
+
location: example.location,
|
35
|
+
file_name: file_name,
|
36
|
+
result: result,
|
37
|
+
failure_reason: failure_reason,
|
38
|
+
failure_expanded: failure_expanded,
|
39
|
+
history: history,
|
40
|
+
).with_indifferent_access.compact
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def file_name
|
46
|
+
@file_name ||= begin
|
47
|
+
identifier_file_name = strip_invalid_utf8_chars(example.id)[FILE_PATH_REGEX]
|
48
|
+
location_file_name = example.location[FILE_PATH_REGEX]
|
49
|
+
|
50
|
+
if identifier_file_name != location_file_name
|
51
|
+
# If the identifier and location files are not the same, we assume
|
52
|
+
# that the test was run as part of a shared example. If this isn't the
|
53
|
+
# case, then there's something we haven't accounted for
|
54
|
+
if shared_example?
|
55
|
+
# Taking the last frame in this backtrace will give us the original
|
56
|
+
# entry point for the shared example
|
57
|
+
shared_example_call_location[FILE_PATH_REGEX]
|
58
|
+
else
|
59
|
+
"Unknown"
|
60
|
+
end
|
61
|
+
else
|
62
|
+
identifier_file_name
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def shared_example?
|
68
|
+
example.metadata[:shared_group_inclusion_backtrace].any?
|
69
|
+
end
|
70
|
+
|
71
|
+
def shared_example_call_location
|
72
|
+
example.metadata[:shared_group_inclusion_backtrace].last.inclusion_location
|
73
|
+
end
|
74
|
+
|
75
|
+
def strip_invalid_utf8_chars(object)
|
76
|
+
if object.is_a?(Hash)
|
77
|
+
Hash[object.map { |key, value| [key, strip_invalid_utf8_chars(value)] }]
|
78
|
+
elsif object.is_a?(Array)
|
79
|
+
object.map { |value| strip_invalid_utf8_chars(value) }
|
80
|
+
elsif object.is_a?(String)
|
81
|
+
object.encode('UTF-8', :invalid => :replace, :undef => :replace)
|
82
|
+
else
|
83
|
+
object
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|