gitlab-qa 5.17.1 → 6.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +1 -1
- data/docs/what_tests_can_be_run.md +1 -2
- data/lib/gitlab/qa.rb +3 -0
- data/lib/gitlab/qa/component/base.rb +12 -12
- data/lib/gitlab/qa/report/json_test_results.rb +21 -0
- data/lib/gitlab/qa/report/junit_test_results.rb +21 -0
- data/lib/gitlab/qa/report/results_in_issues.rb +22 -18
- data/lib/gitlab/qa/report/test_result.rb +54 -0
- data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb +9 -6
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 87249baf75070cf8c5362c34b9995f6163370e9b6e353a393104beefe62d4bd8
|
4
|
+
data.tar.gz: 6e00eee6993c9f5c6aeb4cb187c708941bfd27cb9b59340165b23b02c30f583a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6d621d51ca18fa402e397b13c86dda6d52039a11ca259c6a46f3e9caffeab0faa68b6b9ff58ec4f727b2dae82c386442655122571936a47c366647b3ad895a50
|
7
|
+
data.tar.gz: 73bbcc44b560368e64d8bd335b3fac6d52128e8c05f6ddf582ea6139032f570d80563b6fbc9c96f41598fee2c2c345ec9155f5990f3ea4df1bde2167bede9a27
|
data/.gitlab-ci.yml
CHANGED
@@ -94,7 +94,7 @@ release:
|
|
94
94
|
- exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
95
95
|
- exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
96
96
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
97
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then
|
97
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
98
98
|
- exit $test_run_exit_code
|
99
99
|
|
100
100
|
.ce-qa:
|
@@ -251,8 +251,7 @@ of a Gitaly Cluster.
|
|
251
251
|
|
252
252
|
To run tests against the GitLab container, a GitLab QA (`gitlab/gitlab-qa`)
|
253
253
|
container is spun up and tests are run from it by running the
|
254
|
-
`Test::
|
255
|
-
specifically to test Gitaly Cluster features.
|
254
|
+
`Test::Instance::All` scenario with the `:gitaly_cluster` tag.
|
256
255
|
|
257
256
|
Example:
|
258
257
|
|
data/lib/gitlab/qa.rb
CHANGED
@@ -97,9 +97,12 @@ module Gitlab
|
|
97
97
|
end
|
98
98
|
|
99
99
|
module Report
|
100
|
+
autoload :JsonTestResults, 'gitlab/qa/report/json_test_results'
|
101
|
+
autoload :JUnitTestResults, 'gitlab/qa/report/junit_test_results'
|
100
102
|
autoload :PrepareStageReports, 'gitlab/qa/report/prepare_stage_reports'
|
101
103
|
autoload :ResultsInIssues, 'gitlab/qa/report/results_in_issues'
|
102
104
|
autoload :SummaryTable, 'gitlab/qa/report/summary_table'
|
105
|
+
autoload :TestResult, 'gitlab/qa/report/test_result'
|
103
106
|
autoload :UpdateScreenshotPath, 'gitlab/qa/report/update_screenshot_path'
|
104
107
|
end
|
105
108
|
|
@@ -41,26 +41,16 @@ module Gitlab
|
|
41
41
|
raise NotImplementedError, "#{self.class.name} must specify a docker image tag as DOCKER_IMAGE_TAG"
|
42
42
|
end
|
43
43
|
|
44
|
-
def instance
|
44
|
+
def instance(skip_teardown: false)
|
45
45
|
instance_no_teardown do
|
46
46
|
yield self if block_given?
|
47
47
|
end
|
48
48
|
ensure
|
49
|
-
teardown
|
49
|
+
teardown unless skip_teardown
|
50
50
|
end
|
51
51
|
|
52
52
|
alias_method :launch_and_teardown_instance, :instance
|
53
53
|
|
54
|
-
def instance_no_teardown
|
55
|
-
prepare
|
56
|
-
start
|
57
|
-
reconfigure
|
58
|
-
wait_until_ready
|
59
|
-
process_exec_commands
|
60
|
-
|
61
|
-
yield self if block_given?
|
62
|
-
end
|
63
|
-
|
64
54
|
def prepare
|
65
55
|
prepare_docker_image
|
66
56
|
prepare_network
|
@@ -147,6 +137,16 @@ module Gitlab
|
|
147
137
|
raise 'Invalid instance name!' unless name
|
148
138
|
end
|
149
139
|
|
140
|
+
def instance_no_teardown
|
141
|
+
prepare
|
142
|
+
start
|
143
|
+
reconfigure
|
144
|
+
wait_until_ready
|
145
|
+
process_exec_commands
|
146
|
+
|
147
|
+
yield self if block_given?
|
148
|
+
end
|
149
|
+
|
150
150
|
def teardown?
|
151
151
|
!Runtime::Scenario.attributes.include?(:teardown) || Runtime::Scenario.teardown
|
152
152
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
module QA
|
7
|
+
module Report
|
8
|
+
class JsonTestResults
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
def initialize(file)
|
12
|
+
@testcases = JSON.parse(File.read(file))['examples'].map { |test| TestResult.from_json(test) }
|
13
|
+
end
|
14
|
+
|
15
|
+
def each(&block)
|
16
|
+
@testcases.each(&block)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'nokogiri'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
module QA
|
7
|
+
module Report
|
8
|
+
class JUnitTestResults
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
def initialize(file)
|
12
|
+
@testcases = Nokogiri::XML(File.read(file)).xpath('//testcase').map { |test| TestResult.from_junit(test) }
|
13
|
+
end
|
14
|
+
|
15
|
+
def each(&block)
|
16
|
+
@testcases.each(&block)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -30,11 +30,12 @@ module Gitlab
|
|
30
30
|
RETRY_BACK_OFF_DELAY = 60
|
31
31
|
MAX_RETRY_ATTEMPTS = 3
|
32
32
|
|
33
|
-
def initialize(token:, input_files:, project: nil)
|
33
|
+
def initialize(token:, input_files:, project: nil, input_format: :junit)
|
34
34
|
@token = token
|
35
35
|
@files = Array(input_files)
|
36
36
|
@project = project
|
37
37
|
@retry_backoff = 0
|
38
|
+
@input_format = input_format.to_sym
|
38
39
|
end
|
39
40
|
|
40
41
|
def invoke!
|
@@ -46,7 +47,14 @@ module Gitlab
|
|
46
47
|
|
47
48
|
Dir.glob(files).each do |file|
|
48
49
|
puts "Reporting tests in #{file}"
|
49
|
-
|
50
|
+
case input_format
|
51
|
+
when :json
|
52
|
+
test_results = Report::JsonTestResults.new(file)
|
53
|
+
when :junit
|
54
|
+
test_results = Report::JUnitTestResults.new(file)
|
55
|
+
end
|
56
|
+
|
57
|
+
test_results.each do |test|
|
50
58
|
report_test(test)
|
51
59
|
end
|
52
60
|
end
|
@@ -54,7 +62,7 @@ module Gitlab
|
|
54
62
|
|
55
63
|
private
|
56
64
|
|
57
|
-
attr_reader :files, :token, :project
|
65
|
+
attr_reader :files, :token, :project, :input_format
|
58
66
|
|
59
67
|
def validate_input!
|
60
68
|
assert_project!
|
@@ -99,9 +107,9 @@ module Gitlab
|
|
99
107
|
end
|
100
108
|
|
101
109
|
def report_test(test)
|
102
|
-
return if test.
|
110
|
+
return if test.skipped
|
103
111
|
|
104
|
-
puts "Reporting test: #{test
|
112
|
+
puts "Reporting test: #{test.file} | #{test.name}"
|
105
113
|
|
106
114
|
issue = find_issue(test)
|
107
115
|
if issue
|
@@ -124,7 +132,7 @@ module Gitlab
|
|
124
132
|
Gitlab.create_issue(
|
125
133
|
project,
|
126
134
|
title_from_test(test),
|
127
|
-
{ description: "### Full description\n\n#{search_safe(test
|
135
|
+
{ description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}", labels: 'status::automated' }
|
128
136
|
)
|
129
137
|
end
|
130
138
|
end
|
@@ -135,18 +143,18 @@ module Gitlab
|
|
135
143
|
.auto_paginate
|
136
144
|
.select { |issue| issue.state == 'opened' && issue.title.strip == title_from_test(test) }
|
137
145
|
|
138
|
-
warn(%(Too many issues found with the file path "#{test
|
146
|
+
warn(%(Too many issues found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
|
139
147
|
|
140
148
|
issues.first
|
141
149
|
end
|
142
150
|
end
|
143
151
|
|
144
152
|
def search_term(test)
|
145
|
-
%("#{test
|
153
|
+
%("#{test.file}" "#{search_safe(test.name)}")
|
146
154
|
end
|
147
155
|
|
148
156
|
def title_from_test(test)
|
149
|
-
title = "#{partial_file_path(test
|
157
|
+
title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
|
150
158
|
|
151
159
|
return title unless title.length > MAX_TITLE_LENGTH
|
152
160
|
|
@@ -162,7 +170,7 @@ module Gitlab
|
|
162
170
|
end
|
163
171
|
|
164
172
|
def note_status(issue, test)
|
165
|
-
return if
|
173
|
+
return if test.failures.empty?
|
166
174
|
|
167
175
|
note = note_content(test)
|
168
176
|
|
@@ -176,7 +184,7 @@ module Gitlab
|
|
176
184
|
end
|
177
185
|
|
178
186
|
def note_content(test)
|
179
|
-
errors =
|
187
|
+
errors = test.failures.each_with_object([]) do |failure, text|
|
180
188
|
text << <<~TEXT
|
181
189
|
Error:
|
182
190
|
```
|
@@ -185,7 +193,7 @@ module Gitlab
|
|
185
193
|
|
186
194
|
Stacktrace:
|
187
195
|
```
|
188
|
-
#{failure
|
196
|
+
#{failure['stacktrace']}
|
189
197
|
```
|
190
198
|
TEXT
|
191
199
|
end.join("\n\n")
|
@@ -231,7 +239,7 @@ module Gitlab
|
|
231
239
|
def update_labels(issue, test)
|
232
240
|
labels = issue.labels
|
233
241
|
labels.delete_if { |label| label.start_with?("#{pipeline}::") }
|
234
|
-
labels << (
|
242
|
+
labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
|
235
243
|
labels << "Enterprise Edition" if ee_test?(test)
|
236
244
|
quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
|
237
245
|
|
@@ -242,11 +250,7 @@ module Gitlab
|
|
242
250
|
# rubocop:enable Metrics/AbcSize
|
243
251
|
|
244
252
|
def ee_test?(test)
|
245
|
-
test
|
246
|
-
end
|
247
|
-
|
248
|
-
def failures(test)
|
249
|
-
test.search('failure')
|
253
|
+
test.file =~ %r{features/ee/(api|browser_ui)}
|
250
254
|
end
|
251
255
|
|
252
256
|
def pipeline
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Report
|
6
|
+
class TestResult
|
7
|
+
attr_accessor :name, :file, :skipped, :failures
|
8
|
+
|
9
|
+
def self.from_json(test)
|
10
|
+
new.tap do |test_result|
|
11
|
+
test_result.name = test['full_description']
|
12
|
+
test_result.file = test['file_path']
|
13
|
+
test_result.skipped = test['status'] == 'pending'
|
14
|
+
test_result.failures = failures_from_json_exceptions(test)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.from_junit(test)
|
19
|
+
new.tap do |test_result|
|
20
|
+
test_result.name = test['name']
|
21
|
+
test_result.file = test['file']
|
22
|
+
test_result.skipped = test.search('skipped').any?
|
23
|
+
test_result.failures = failures_from_junit_exceptions(test)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.failures_from_json_exceptions(test)
|
28
|
+
return [] unless test.key?('exceptions')
|
29
|
+
|
30
|
+
test['exceptions'].map do |exception|
|
31
|
+
{
|
32
|
+
'message' => "#{exception['class']}: #{exception['message']}",
|
33
|
+
'stacktrace' => exception['backtrace'].join('\n')
|
34
|
+
}
|
35
|
+
end
|
36
|
+
end
|
37
|
+
private_class_method :failures_from_json_exceptions
|
38
|
+
|
39
|
+
def self.failures_from_junit_exceptions(test)
|
40
|
+
failures = test.search('failure')
|
41
|
+
return [] if failures.empty?
|
42
|
+
|
43
|
+
failures.map do |exception|
|
44
|
+
{
|
45
|
+
'message' => "#{exception['type']}: #{exception['message']}",
|
46
|
+
'stacktrace' => exception.content
|
47
|
+
}
|
48
|
+
end
|
49
|
+
end
|
50
|
+
private_class_method :failures_from_junit_exceptions
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -7,13 +7,13 @@ module Gitlab
|
|
7
7
|
attr_reader :gitlab_name, :spec_suite
|
8
8
|
|
9
9
|
def initialize
|
10
|
-
@gitlab_name = 'gitlab-gitaly-
|
10
|
+
@gitlab_name = 'gitlab-gitaly-cluster'
|
11
11
|
@primary_node_name = 'gitaly1'
|
12
12
|
@secondary_node_name = 'gitaly2'
|
13
13
|
@tertiary_node_name = 'gitaly3'
|
14
14
|
@praefect_node_name = 'praefect'
|
15
15
|
@database = 'postgres'
|
16
|
-
@spec_suite = 'Test::
|
16
|
+
@spec_suite = 'Test::Instance::All'
|
17
17
|
@network = 'test'
|
18
18
|
end
|
19
19
|
|
@@ -26,7 +26,7 @@ module Gitlab
|
|
26
26
|
sql_node = Component::PostgreSQL.new.tap do |sql|
|
27
27
|
sql.name = @database
|
28
28
|
sql.network = @network
|
29
|
-
sql.
|
29
|
+
sql.instance(skip_teardown: true) do
|
30
30
|
sql.run_psql '-d template1 -c "CREATE DATABASE praefect_production OWNER postgres"'
|
31
31
|
end
|
32
32
|
end
|
@@ -39,7 +39,7 @@ module Gitlab
|
|
39
39
|
|
40
40
|
praefect.omnibus_config = praefect_omnibus_configuration
|
41
41
|
|
42
|
-
praefect.
|
42
|
+
praefect.instance(skip_teardown: true)
|
43
43
|
end
|
44
44
|
|
45
45
|
Component::Gitlab.perform do |gitlab|
|
@@ -49,7 +49,10 @@ module Gitlab
|
|
49
49
|
|
50
50
|
gitlab.omnibus_config = gitlab_omnibus_configuration
|
51
51
|
gitlab.instance do
|
52
|
-
puts "Running Gitaly
|
52
|
+
puts "Running Gitaly Cluster specs!"
|
53
|
+
|
54
|
+
rspec_args << "--" unless rspec_args.include?('--')
|
55
|
+
rspec_args << %w[--tag gitaly_cluster]
|
53
56
|
|
54
57
|
Component::Specs.perform do |specs|
|
55
58
|
specs.suite = spec_suite
|
@@ -190,7 +193,7 @@ module Gitlab
|
|
190
193
|
gitaly.network = @network
|
191
194
|
gitaly.skip_availability_check = true
|
192
195
|
gitaly.omnibus_config = gitaly_omnibus_configuration
|
193
|
-
gitaly.
|
196
|
+
gitaly.instance(skip_teardown: true)
|
194
197
|
end
|
195
198
|
end
|
196
199
|
end
|
data/lib/gitlab/qa/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 6.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Grzegorz Bizon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-08-
|
11
|
+
date: 2020-08-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -257,9 +257,12 @@ files:
|
|
257
257
|
- lib/gitlab/qa/docker/shellout.rb
|
258
258
|
- lib/gitlab/qa/docker/volumes.rb
|
259
259
|
- lib/gitlab/qa/release.rb
|
260
|
+
- lib/gitlab/qa/report/json_test_results.rb
|
261
|
+
- lib/gitlab/qa/report/junit_test_results.rb
|
260
262
|
- lib/gitlab/qa/report/prepare_stage_reports.rb
|
261
263
|
- lib/gitlab/qa/report/results_in_issues.rb
|
262
264
|
- lib/gitlab/qa/report/summary_table.rb
|
265
|
+
- lib/gitlab/qa/report/test_result.rb
|
263
266
|
- lib/gitlab/qa/report/update_screenshot_path.rb
|
264
267
|
- lib/gitlab/qa/reporter.rb
|
265
268
|
- lib/gitlab/qa/runner.rb
|