gitlab-qa 6.5.0 → 6.10.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +67 -3
- data/.rubocop.yml +2 -0
- data/.rubocop_todo.yml +98 -0
- data/docs/what_tests_can_be_run.md +17 -0
- data/gitlab-qa.gemspec +5 -5
- data/lib/gitlab/qa.rb +4 -0
- data/lib/gitlab/qa/component/base.rb +8 -1
- data/lib/gitlab/qa/docker/engine.rb +8 -0
- data/lib/gitlab/qa/docker/shellout.rb +1 -1
- data/lib/gitlab/qa/report/base_test_results.rb +6 -3
- data/lib/gitlab/qa/report/generate_test_session.rb +203 -0
- data/lib/gitlab/qa/report/gitlab_issue_client.rb +21 -7
- data/lib/gitlab/qa/report/gitlab_issue_dry_client.rb +28 -0
- data/lib/gitlab/qa/report/json_test_results.rb +2 -2
- data/lib/gitlab/qa/report/junit_test_results.rb +2 -2
- data/lib/gitlab/qa/report/relate_failure_issue.rb +143 -0
- data/lib/gitlab/qa/report/report_as_issue.rb +101 -3
- data/lib/gitlab/qa/report/results_in_issues.rb +26 -92
- data/lib/gitlab/qa/report/test_result.rb +31 -1
- data/lib/gitlab/qa/reporter.rb +30 -2
- data/lib/gitlab/qa/runtime/env.rb +31 -3
- data/lib/gitlab/qa/scenario/test/integration/actioncable.rb +36 -0
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +19 -14
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
|
3
5
|
module Gitlab
|
4
6
|
module QA
|
5
7
|
module Report
|
@@ -19,6 +21,10 @@ module Gitlab
|
|
19
21
|
self.failures = failures_from_exceptions
|
20
22
|
end
|
21
23
|
|
24
|
+
def stage
|
25
|
+
@stage ||= file[%r{(?:api|browser_ui)/(?:(?:\d+_)?(\w+))}, 1]
|
26
|
+
end
|
27
|
+
|
22
28
|
def name
|
23
29
|
raise NotImplementedError
|
24
30
|
end
|
@@ -46,8 +52,16 @@ module Gitlab
|
|
46
52
|
report['file_path']
|
47
53
|
end
|
48
54
|
|
55
|
+
def status
|
56
|
+
report['status']
|
57
|
+
end
|
58
|
+
|
59
|
+
def ci_job_url
|
60
|
+
report['ci_job_url']
|
61
|
+
end
|
62
|
+
|
49
63
|
def skipped
|
50
|
-
|
64
|
+
status == 'pending'
|
51
65
|
end
|
52
66
|
|
53
67
|
def testcase
|
@@ -58,6 +72,21 @@ module Gitlab
|
|
58
72
|
report['testcase'] = new_testcase
|
59
73
|
end
|
60
74
|
|
75
|
+
def failure_issue
|
76
|
+
report['failure_issue']
|
77
|
+
end
|
78
|
+
|
79
|
+
def failure_issue=(new_failure_issue)
|
80
|
+
report['failure_issue'] = new_failure_issue
|
81
|
+
end
|
82
|
+
|
83
|
+
def quarantine?
|
84
|
+
# The value for 'quarantine' could be nil, a hash, a string,
|
85
|
+
# or true (if the test just has the :quarantine tag)
|
86
|
+
# But any non-nil or false value should means the test is in quarantine
|
87
|
+
report['quarantine'].present?
|
88
|
+
end
|
89
|
+
|
61
90
|
private
|
62
91
|
|
63
92
|
# rubocop:disable Metrics/AbcSize
|
@@ -71,6 +100,7 @@ module Gitlab
|
|
71
100
|
|
72
101
|
{
|
73
102
|
'message' => "#{exception['class']}: #{exception['message']}",
|
103
|
+
'message_lines' => exception['message_lines'],
|
74
104
|
'stacktrace' => "#{exception['message_lines'].join("\n")}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}"
|
75
105
|
}
|
76
106
|
end
|
data/lib/gitlab/qa/reporter.rb
CHANGED
@@ -4,6 +4,7 @@ module Gitlab
|
|
4
4
|
module QA
|
5
5
|
class Reporter
|
6
6
|
# rubocop:disable Metrics/AbcSize
|
7
|
+
# rubocop:disable Metrics/PerceivedComplexity
|
7
8
|
def self.invoke(args)
|
8
9
|
report_options = {}
|
9
10
|
slack_options = {}
|
@@ -21,11 +22,25 @@ module Gitlab
|
|
21
22
|
report_options[:input_files] = files if files
|
22
23
|
end
|
23
24
|
|
24
|
-
opts.on('-
|
25
|
+
opts.on('--relate-failure-issue FILES', String, 'Relate test failures to failure issues from RSpec JSON files') do |files|
|
26
|
+
report_options[:relate_failure_issue] = true
|
27
|
+
report_options[:input_files] = files if files
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on('--max-diff-ratio DIFF_RATO', Float, 'Max stacktrace diff ratio for QA failure issues detection. Used by with --relate-failure-issue') do |value|
|
31
|
+
report_options[:max_diff_ratio] = value
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on('-p', '--project PROJECT_ID', String, 'A valid project ID. Can be an integer or a group/project string. Required by --report-in-issues and --relate-failure-issue') do |value|
|
25
35
|
report_options[:project] = value
|
26
36
|
end
|
27
37
|
|
28
|
-
opts.on('-
|
38
|
+
opts.on('--generate-test-session FILES', String, 'Generate test session report') do |files|
|
39
|
+
report_options[:generate_test_session] = true
|
40
|
+
report_options[:input_files] = files if files
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token. Required by --report-in-issues and --relate-failure-issue') do |value|
|
29
44
|
report_options[:token] = value
|
30
45
|
end
|
31
46
|
|
@@ -45,6 +60,10 @@ module Gitlab
|
|
45
60
|
report_options[:files] = files
|
46
61
|
end
|
47
62
|
|
63
|
+
opts.on('--dry-run', "Perform a dry-run (don't create or update issues)") do |files|
|
64
|
+
report_options[:dry_run] = true
|
65
|
+
end
|
66
|
+
|
48
67
|
opts.on_tail('-v', '--version', 'Show the version') do
|
49
68
|
require 'gitlab/qa/version'
|
50
69
|
puts "#{$PROGRAM_NAME} : #{VERSION}"
|
@@ -63,10 +82,18 @@ module Gitlab
|
|
63
82
|
if report_options.delete(:prepare_stage_reports)
|
64
83
|
Gitlab::QA::Report::PrepareStageReports.new(**report_options).invoke!
|
65
84
|
|
85
|
+
elsif report_options.delete(:relate_failure_issue)
|
86
|
+
report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
|
87
|
+
Gitlab::QA::Report::RelateFailureIssue.new(**report_options).invoke!
|
88
|
+
|
66
89
|
elsif report_options.delete(:report_in_issues)
|
67
90
|
report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
|
68
91
|
Gitlab::QA::Report::ResultsInIssues.new(**report_options).invoke!
|
69
92
|
|
93
|
+
elsif report_options.delete(:generate_test_session)
|
94
|
+
report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
|
95
|
+
Gitlab::QA::Report::GenerateTestSession.new(**report_options).invoke!
|
96
|
+
|
70
97
|
elsif slack_options.delete(:post_to_slack)
|
71
98
|
Gitlab::QA::Slack::PostToSlack.new(**slack_options).invoke!
|
72
99
|
|
@@ -79,6 +106,7 @@ module Gitlab
|
|
79
106
|
exit 1
|
80
107
|
end
|
81
108
|
end
|
109
|
+
# rubocop:enable Metrics/PerceivedComplexity
|
82
110
|
# rubocop:enable Metrics/AbcSize
|
83
111
|
end
|
84
112
|
end
|
@@ -8,6 +8,7 @@ module Gitlab
|
|
8
8
|
|
9
9
|
# Variables that are used in tests and are passed through to the docker container that executes the tests.
|
10
10
|
# These variables should be listed in /docs/what_tests_can_be_run.md#supported-gitlab-environment-variables
|
11
|
+
# unless they're defined elsewhere (e.g.: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)
|
11
12
|
ENV_VARIABLES = {
|
12
13
|
'QA_REMOTE_GRID' => :remote_grid,
|
13
14
|
'QA_REMOTE_GRID_USERNAME' => :remote_grid_username,
|
@@ -62,13 +63,13 @@ module Gitlab
|
|
62
63
|
'KNAPSACK_TEST_FILE_PATTERN' => :knapsack_test_file_pattern,
|
63
64
|
'KNAPSACK_TEST_DIR' => :knapsack_test_dir,
|
64
65
|
'CI' => :ci,
|
65
|
-
'CI_JOB_ID' => :ci_job_id,
|
66
66
|
'CI_JOB_URL' => :ci_job_url,
|
67
67
|
'CI_RUNNER_ID' => :ci_runner_id,
|
68
68
|
'CI_SERVER_HOST' => :ci_server_host,
|
69
69
|
'CI_SERVER_PERSONAL_ACCESS_TOKEN' => :ci_server_personal_access_token,
|
70
70
|
'CI_NODE_INDEX' => :ci_node_index,
|
71
71
|
'CI_NODE_TOTAL' => :ci_node_total,
|
72
|
+
'CI_PROJECT_NAME' => :ci_project_name,
|
72
73
|
'GITLAB_CI' => :gitlab_ci,
|
73
74
|
'QA_SKIP_PULL' => :qa_skip_pull,
|
74
75
|
'ELASTIC_URL' => :elastic_url,
|
@@ -91,7 +92,10 @@ module Gitlab
|
|
91
92
|
attr_writer(method_name)
|
92
93
|
|
93
94
|
define_method(method_name) do
|
94
|
-
ENV[env_name] ||
|
95
|
+
ENV[env_name] ||
|
96
|
+
if instance_variable_defined?("@#{method_name}")
|
97
|
+
instance_variable_get("@#{method_name}")
|
98
|
+
end
|
95
99
|
end
|
96
100
|
end
|
97
101
|
|
@@ -123,12 +127,28 @@ module Gitlab
|
|
123
127
|
ENV['CI_PIPELINE_SOURCE']
|
124
128
|
end
|
125
129
|
|
130
|
+
def ci_pipeline_url
|
131
|
+
ENV['CI_PIPELINE_URL']
|
132
|
+
end
|
133
|
+
|
134
|
+
def ci_pipeline_id
|
135
|
+
ENV['CI_PIPELINE_ID']
|
136
|
+
end
|
137
|
+
|
126
138
|
def ci_project_name
|
127
139
|
ENV['CI_PROJECT_NAME']
|
128
140
|
end
|
129
141
|
|
130
142
|
def pipeline_from_project_name
|
131
|
-
ci_project_name.to_s.start_with?('gitlab-qa')
|
143
|
+
if ci_project_name.to_s.start_with?('gitlab-qa')
|
144
|
+
if ENV['TOP_UPSTREAM_SOURCE_JOB'].to_s.start_with?('https://ops.gitlab.net')
|
145
|
+
'staging-orchestrated'
|
146
|
+
else
|
147
|
+
'master'
|
148
|
+
end
|
149
|
+
else
|
150
|
+
ci_project_name
|
151
|
+
end
|
132
152
|
end
|
133
153
|
|
134
154
|
def run_id
|
@@ -147,6 +167,14 @@ module Gitlab
|
|
147
167
|
ENV['GITLAB_QA_CONTAINER_REGISTRY_ACCESS_TOKEN']
|
148
168
|
end
|
149
169
|
|
170
|
+
def qa_issue_url
|
171
|
+
ENV['GITLAB_QA_ISSUE_URL']
|
172
|
+
end
|
173
|
+
|
174
|
+
def deploy_environment
|
175
|
+
ENV['DEPLOY_ENVIRONMENT'] || pipeline_from_project_name
|
176
|
+
end
|
177
|
+
|
150
178
|
def host_artifacts_dir
|
151
179
|
@host_artifacts_dir ||= File.join(ENV['QA_ARTIFACTS_DIR'] || '/tmp/gitlab-qa', Runtime::Env.run_id)
|
152
180
|
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Gitlab
|
2
|
+
module QA
|
3
|
+
module Scenario
|
4
|
+
module Test
|
5
|
+
module Integration
|
6
|
+
class Actioncable < Scenario::Template
|
7
|
+
def perform(release, *rspec_args)
|
8
|
+
Component::Gitlab.perform do |gitlab|
|
9
|
+
gitlab.release = QA::Release.new(release)
|
10
|
+
gitlab.name = 'gitlab-actioncable'
|
11
|
+
gitlab.network = 'test'
|
12
|
+
gitlab.omnibus_config = <<~OMNIBUS
|
13
|
+
actioncable['enable'] = true;
|
14
|
+
OMNIBUS
|
15
|
+
|
16
|
+
gitlab.instance do
|
17
|
+
puts "Running actioncable specs!"
|
18
|
+
|
19
|
+
rspec_args << "--" unless rspec_args.include?('--')
|
20
|
+
rspec_args << %w[--tag actioncable]
|
21
|
+
|
22
|
+
Component::Specs.perform do |specs|
|
23
|
+
specs.suite = 'Test::Instance::All'
|
24
|
+
specs.release = gitlab.release
|
25
|
+
specs.network = gitlab.network
|
26
|
+
specs.args = [gitlab.address, *rspec_args]
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
data/lib/gitlab/qa/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.
|
4
|
+
version: 6.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Grzegorz Bizon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-11-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -28,16 +28,16 @@ dependencies:
|
|
28
28
|
name: gitlab-styles
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 4.3.0
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 4.3.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: pry
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,28 +86,28 @@ dependencies:
|
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: 0.
|
89
|
+
version: 0.82.0
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: 0.
|
96
|
+
version: 0.82.0
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: rubocop-rspec
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: 1.
|
103
|
+
version: '1.36'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- -
|
108
|
+
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: 1.
|
110
|
+
version: '1.36'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: webmock
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -142,14 +142,14 @@ dependencies:
|
|
142
142
|
requirements:
|
143
143
|
- - "~>"
|
144
144
|
- !ruby/object:Gem::Version
|
145
|
-
version: 4.
|
145
|
+
version: 4.16.1
|
146
146
|
type: :runtime
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
150
|
- - "~>"
|
151
151
|
- !ruby/object:Gem::Version
|
152
|
-
version: 4.
|
152
|
+
version: 4.16.1
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: http
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
@@ -207,6 +207,7 @@ files:
|
|
207
207
|
- ".gitlab/merge_request_templates/Release.md"
|
208
208
|
- ".rspec"
|
209
209
|
- ".rubocop.yml"
|
210
|
+
- ".rubocop_todo.yml"
|
210
211
|
- ".travis.yml"
|
211
212
|
- CONTRIBUTING.md
|
212
213
|
- Gemfile
|
@@ -258,10 +259,13 @@ files:
|
|
258
259
|
- lib/gitlab/qa/docker/volumes.rb
|
259
260
|
- lib/gitlab/qa/release.rb
|
260
261
|
- lib/gitlab/qa/report/base_test_results.rb
|
262
|
+
- lib/gitlab/qa/report/generate_test_session.rb
|
261
263
|
- lib/gitlab/qa/report/gitlab_issue_client.rb
|
264
|
+
- lib/gitlab/qa/report/gitlab_issue_dry_client.rb
|
262
265
|
- lib/gitlab/qa/report/json_test_results.rb
|
263
266
|
- lib/gitlab/qa/report/junit_test_results.rb
|
264
267
|
- lib/gitlab/qa/report/prepare_stage_reports.rb
|
268
|
+
- lib/gitlab/qa/report/relate_failure_issue.rb
|
265
269
|
- lib/gitlab/qa/report/report_as_issue.rb
|
266
270
|
- lib/gitlab/qa/report/results_in_issues.rb
|
267
271
|
- lib/gitlab/qa/report/summary_table.rb
|
@@ -288,6 +292,7 @@ files:
|
|
288
292
|
- lib/gitlab/qa/scenario/test/instance/smoke.rb
|
289
293
|
- lib/gitlab/qa/scenario/test/instance/staging.rb
|
290
294
|
- lib/gitlab/qa/scenario/test/instance/staging_geo.rb
|
295
|
+
- lib/gitlab/qa/scenario/test/integration/actioncable.rb
|
291
296
|
- lib/gitlab/qa/scenario/test/integration/elasticsearch.rb
|
292
297
|
- lib/gitlab/qa/scenario/test/integration/geo.rb
|
293
298
|
- lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb
|