gitlab-qa 7.24.4 → 7.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab/ci/rules.gitlab-ci.yml +19 -6
- data/.gitlab/issue_templates/Default.md +13 -0
- data/.gitlab/merge_request_templates/Default.md +46 -0
- data/.gitlab-ci.yml +115 -1331
- data/Dangerfile +1 -3
- data/docs/what_tests_can_be_run.md +15 -0
- data/gitlab-qa.gemspec +1 -1
- data/lib/gitlab/qa/component/gitlab.rb +35 -0
- data/lib/gitlab/qa/component/staging_ref.rb +1 -0
- data/lib/gitlab/qa/docker/engine.rb +4 -0
- data/lib/gitlab/qa/release.rb +1 -1
- data/lib/gitlab/qa/report/gitlab_issue_client.rb +12 -0
- data/lib/gitlab/qa/report/relate_failure_issue.rb +27 -5
- data/lib/gitlab/qa/runner.rb +10 -0
- data/lib/gitlab/qa/runtime/env.rb +13 -0
- data/lib/gitlab/qa/scenario/test/instance/staging_ref_geo.rb +27 -0
- data/lib/gitlab/qa/scenario/test/integration/registry_with_cdn.rb +42 -0
- data/lib/gitlab/qa/version.rb +1 -1
- data/lib/gitlab/qa.rb +2 -0
- data/scripts/generate-qa-jobs.rb +1741 -0
- data/support/data/license_usage_seed.rb +75 -0
- metadata +10 -5
- data/.gitlab/ci/danger.gitlab-ci.yml +0 -25
data/Dangerfile
CHANGED
@@ -1,9 +1,7 @@
|
|
1
1
|
require 'gitlab-dangerfiles'
|
2
2
|
|
3
3
|
Gitlab::Dangerfiles.for_project(self) do |dangerfiles|
|
4
|
-
# Import all plugins from the gem
|
5
4
|
dangerfiles.import_plugins
|
6
5
|
|
7
|
-
|
8
|
-
dangerfiles.import_dangerfiles(only: %w[changes_size commit_messages simple_roulette])
|
6
|
+
dangerfiles.import_dangerfiles(except: %w[changelog])
|
9
7
|
end
|
@@ -818,6 +818,21 @@ $ export GITLAB_PASSWORD="$GITLAB_QA_PASSWORD"
|
|
818
818
|
$ gitlab-qa Test::Instance::StagingGeo
|
819
819
|
```
|
820
820
|
|
821
|
+
### `Test::Instance::StagingRefGeo`
|
822
|
+
|
823
|
+
This scenario tests that the Geo staging deployment (with [`staging-ref.gitlab.com`](https://staging-ref.gitlab.com) as the primary site and [`geo.staging-ref.gitlab.com`](https://geo.staging-ref.gitlab.com) as the secondary site) works as expected by running tests tagged `:geo` against it. This is done by spinning up a GitLab QA (`gitlab/gitlab-qa`) container and running the `QA::EE::Scenario::Test::Geo` scenario. Note that the Geo setup steps in the `QA::EE::Scenario::Test::Geo` scenario are skipped when testing a live Geo deployment.
|
824
|
+
|
825
|
+
Scenario requirements are the same as [`Test::Instance::StagingGeo`](#testinstancestaginggeo) described above.
|
826
|
+
|
827
|
+
```
|
828
|
+
$ export GITLAB_QA_ACCESS_TOKEN=your_api_access_token
|
829
|
+
$ export GITLAB_QA_DEV_ACCESS_TOKEN=your_dev_registry_access_token
|
830
|
+
$ export GITLAB_USERNAME="gitlab-qa"
|
831
|
+
$ export GITLAB_PASSWORD="$GITLAB_QA_PASSWORD"
|
832
|
+
|
833
|
+
$ gitlab-qa Test::Instance::StagingRefGeo
|
834
|
+
```
|
835
|
+
|
821
836
|
### `Test::Instance::Production`
|
822
837
|
|
823
838
|
This scenario functions the same as `Test::Instance::Staging`
|
data/gitlab-qa.gemspec
CHANGED
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.require_paths = ['lib']
|
22
22
|
|
23
23
|
spec.add_development_dependency 'climate_control', '~> 1.0.1'
|
24
|
-
spec.add_development_dependency 'gitlab-dangerfiles', '~> 2.
|
24
|
+
spec.add_development_dependency 'gitlab-dangerfiles', '~> 2.11'
|
25
25
|
spec.add_development_dependency 'gitlab-styles', '~> 6.2.1'
|
26
26
|
spec.add_development_dependency 'pry', '~> 0.11'
|
27
27
|
spec.add_development_dependency 'rake', '~> 13.0'
|
@@ -5,6 +5,7 @@ require 'net/http'
|
|
5
5
|
require 'uri'
|
6
6
|
require 'forwardable'
|
7
7
|
require 'openssl'
|
8
|
+
require 'tempfile'
|
8
9
|
|
9
10
|
module Gitlab
|
10
11
|
module QA
|
@@ -19,9 +20,11 @@ module Gitlab
|
|
19
20
|
def_delegators :release, :tag, :image, :edition
|
20
21
|
|
21
22
|
CERTIFICATES_PATH = File.expand_path('../../../../tls_certificates', __dir__)
|
23
|
+
DATA_SEED_PATH = File.expand_path('../../../../support/data', __dir__)
|
22
24
|
|
23
25
|
SSL_PATH = '/etc/gitlab/ssl'
|
24
26
|
TRUSTED_PATH = '/etc/gitlab/trusted-certs'
|
27
|
+
DATA_PATH = '/tmp/data-seeds'
|
25
28
|
|
26
29
|
def initialize
|
27
30
|
super
|
@@ -40,6 +43,7 @@ module Gitlab
|
|
40
43
|
@volumes[@authority_cert_path] = TRUSTED_PATH
|
41
44
|
|
42
45
|
self.release = 'CE'
|
46
|
+
self.exec_commands += seed_test_data_command if Runtime::Scenario.seed_db
|
43
47
|
end
|
44
48
|
|
45
49
|
def set_formless_login_token
|
@@ -166,6 +170,12 @@ module Gitlab
|
|
166
170
|
end
|
167
171
|
end
|
168
172
|
|
173
|
+
def process_exec_commands
|
174
|
+
@docker.copy(name, DATA_SEED_PATH, DATA_PATH) if Runtime::Scenario.seed_db
|
175
|
+
|
176
|
+
exec_commands.each { |command| @docker.exec(name, command) }
|
177
|
+
end
|
178
|
+
|
169
179
|
def sha_version
|
170
180
|
json = @docker.read_file(
|
171
181
|
@release.image, @release.tag,
|
@@ -176,6 +186,19 @@ module Gitlab
|
|
176
186
|
manifest['software']['gitlab-rails']['locked_version']
|
177
187
|
end
|
178
188
|
|
189
|
+
def copy_key_file(env_key)
|
190
|
+
key_dir = ENV['CI_PROJECT_DIR'] || Dir.tmpdir
|
191
|
+
key_file = Tempfile.new(env_key.downcase, key_dir)
|
192
|
+
key_file.write(ENV.fetch(env_key))
|
193
|
+
key_file.close
|
194
|
+
|
195
|
+
File.chmod(0o744, key_file.path)
|
196
|
+
|
197
|
+
@volumes[key_file.path] = key_file.path
|
198
|
+
|
199
|
+
key_file.path
|
200
|
+
end
|
201
|
+
|
179
202
|
private
|
180
203
|
|
181
204
|
# Copy certs to a temporary directory in current working directory.
|
@@ -196,6 +219,18 @@ module Gitlab
|
|
196
219
|
end
|
197
220
|
end
|
198
221
|
|
222
|
+
def seed_test_data_command
|
223
|
+
cmd = []
|
224
|
+
|
225
|
+
Runtime::Scenario.seed_db.each do |file_patterns|
|
226
|
+
Dir["#{DATA_SEED_PATH}/#{file_patterns}"].map { |f| File.basename f }.each do |file|
|
227
|
+
cmd << "gitlab-rails runner #{DATA_PATH}/#{file}"
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
cmd.uniq
|
232
|
+
end
|
233
|
+
|
199
234
|
class Availability
|
200
235
|
def initialize(name, relative_path: '', scheme: 'http', protocol_port: 80)
|
201
236
|
@docker = Docker::Engine.new
|
@@ -76,6 +76,10 @@ module Gitlab
|
|
76
76
|
Docker::Command.execute("attach --sig-proxy=false #{name}", &block)
|
77
77
|
end
|
78
78
|
|
79
|
+
def copy(name, src_path, dest_path)
|
80
|
+
Docker::Command.execute("cp #{src_path} #{name}:#{dest_path}")
|
81
|
+
end
|
82
|
+
|
79
83
|
def restart(name)
|
80
84
|
Docker::Command.execute("restart #{name}")
|
81
85
|
end
|
data/lib/gitlab/qa/release.rb
CHANGED
@@ -147,7 +147,7 @@ module Gitlab
|
|
147
147
|
registry: DEV_REGISTRY
|
148
148
|
}
|
149
149
|
elsif omnibus_mirror?
|
150
|
-
username, password = if Runtime::Env.ci_job_token && Runtime::Env.ci_pipeline_source
|
150
|
+
username, password = if Runtime::Env.ci_job_token && Runtime::Env.ci_pipeline_source.include?('pipeline')
|
151
151
|
['gitlab-ci-token', Runtime::Env.ci_job_token]
|
152
152
|
elsif Runtime::Env.qa_container_registry_access_token
|
153
153
|
[Runtime::Env.gitlab_username, Runtime::Env.qa_container_registry_access_token]
|
@@ -77,12 +77,24 @@ module Gitlab
|
|
77
77
|
end
|
78
78
|
end
|
79
79
|
|
80
|
+
def find_issue_notes(iid:)
|
81
|
+
handle_gitlab_client_exceptions do
|
82
|
+
Gitlab.issue_notes(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
80
86
|
def create_issue_note(iid:, note:)
|
81
87
|
handle_gitlab_client_exceptions do
|
82
88
|
Gitlab.create_issue_note(project, iid, note)
|
83
89
|
end
|
84
90
|
end
|
85
91
|
|
92
|
+
def edit_issue_note(issue_iid:, note_id:, note:)
|
93
|
+
handle_gitlab_client_exceptions do
|
94
|
+
Gitlab.edit_issue_note(project, issue_iid, note_id, note)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
86
98
|
def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
|
87
99
|
handle_gitlab_client_exceptions do
|
88
100
|
Gitlab.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: body)
|
@@ -61,7 +61,7 @@ module Gitlab
|
|
61
61
|
|
62
62
|
if issue
|
63
63
|
puts " => Found issue #{issue.web_url} for test '#{test.name}' with a diff ratio of #{(diff_ratio * 100).round(2)}%."
|
64
|
-
@commented_issue_list.include?(issue.web_url) ? (puts " => Failure already commented on issue.") :
|
64
|
+
@commented_issue_list.include?(issue.web_url) ? (puts " => Failure already commented on issue.") : post_or_update_failed_job_note(issue, test)
|
65
65
|
@commented_issue_list.add(issue.web_url)
|
66
66
|
end
|
67
67
|
|
@@ -86,14 +86,16 @@ module Gitlab
|
|
86
86
|
ld = Class.new.extend(Gem::Text).method(:levenshtein_distance)
|
87
87
|
full_stacktrace = test.failures.first['message_lines'].join("\n")
|
88
88
|
first_test_failure_stacktrace = sanitize_stacktrace(full_stacktrace, FAILURE_STACKTRACE_REGEX) || full_stacktrace
|
89
|
+
clean_first_test_failure_stacktrace = remove_unique_resource_names(first_test_failure_stacktrace)
|
89
90
|
|
90
91
|
# Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
|
91
92
|
failure_issues(test).each_with_object({}) do |issue, memo|
|
92
93
|
relevant_issue_stacktrace = find_issue_stacktrace(issue)
|
93
94
|
next unless relevant_issue_stacktrace
|
94
95
|
|
95
|
-
|
96
|
-
|
96
|
+
clean_relevant_issue_stacktrace = remove_unique_resource_names(relevant_issue_stacktrace)
|
97
|
+
distance = ld.call(clean_first_test_failure_stacktrace, clean_relevant_issue_stacktrace)
|
98
|
+
diff_ratio = distance.zero? ? 0.0 : (distance.to_f / clean_first_test_failure_stacktrace.size).round(3)
|
97
99
|
|
98
100
|
if diff_ratio <= max_diff_ratio
|
99
101
|
puts " => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
|
@@ -128,6 +130,10 @@ module Gitlab
|
|
128
130
|
end
|
129
131
|
end
|
130
132
|
|
133
|
+
def remove_unique_resource_names(stacktrace)
|
134
|
+
stacktrace.gsub(/qa-(test|user)-[a-z0-9-]+/, '<unique-test-resource>').gsub(/(?:-|_)(?:\d+[a-z]|[a-z]+\d)[a-z\d]{4,}/, '<unique-hash>')
|
135
|
+
end
|
136
|
+
|
131
137
|
def find_failure_issue(test)
|
132
138
|
relevant_issues = find_relevant_failure_issues(test)
|
133
139
|
|
@@ -164,14 +170,30 @@ module Gitlab
|
|
164
170
|
super << pipeline_name_label
|
165
171
|
end
|
166
172
|
|
167
|
-
def
|
168
|
-
|
173
|
+
def post_or_update_failed_job_note(issue, test)
|
174
|
+
current_note = "Failed most recently in #{pipeline} pipeline: #{test.ci_job_url}"
|
175
|
+
existing_note = existing_failure_note(issue)
|
176
|
+
|
177
|
+
if existing_note
|
178
|
+
gitlab.edit_issue_note(issue_iid: issue.iid, note_id: existing_note.id, note: current_note)
|
179
|
+
else
|
180
|
+
gitlab.create_issue_note(iid: issue.iid, note: current_note)
|
181
|
+
end
|
182
|
+
|
169
183
|
puts " => Linked #{test.ci_job_url} to #{issue.web_url}."
|
170
184
|
end
|
171
185
|
|
172
186
|
def new_issue_title(test)
|
173
187
|
"Failure in #{super}"
|
174
188
|
end
|
189
|
+
|
190
|
+
def existing_failure_note(issue)
|
191
|
+
gitlab.find_issue_notes(iid: issue.iid).each do |note|
|
192
|
+
return note if note.body.include?('Failed most recently in')
|
193
|
+
end
|
194
|
+
|
195
|
+
false
|
196
|
+
end
|
175
197
|
end
|
176
198
|
end
|
177
199
|
end
|
data/lib/gitlab/qa/runner.rb
CHANGED
@@ -12,9 +12,11 @@ module Gitlab
|
|
12
12
|
Runtime::Scenario.define(:run_tests, true)
|
13
13
|
Runtime::Scenario.define(:qa_image, Runtime::Env.qa_image) if Runtime::Env.qa_image
|
14
14
|
Runtime::Scenario.define(:omnibus_configuration, Runtime::OmnibusConfiguration.new)
|
15
|
+
Runtime::Scenario.define(:seed_db, false)
|
15
16
|
|
16
17
|
# Omnibus Configurators specified by flags
|
17
18
|
@active_configurators = []
|
19
|
+
@seed_scripts = []
|
18
20
|
@omnibus_configurations = %w[default] # always load default configuration
|
19
21
|
|
20
22
|
@options = OptionParser.new do |opts|
|
@@ -45,6 +47,14 @@ module Gitlab
|
|
45
47
|
end
|
46
48
|
end
|
47
49
|
|
50
|
+
opts.on('--seed-db search_pattern1[,search_pattern2,...]', 'Seed application database with sample test data') do |file_pattern|
|
51
|
+
file_pattern.split(',').each do |pattern|
|
52
|
+
@seed_scripts << pattern
|
53
|
+
end
|
54
|
+
|
55
|
+
Runtime::Scenario.define(:seed_db, @seed_scripts)
|
56
|
+
end
|
57
|
+
|
48
58
|
opts.on_tail('-h', '--help', 'Show the usage') do
|
49
59
|
puts opts
|
50
60
|
exit
|
@@ -127,6 +127,11 @@ module Gitlab
|
|
127
127
|
'GOOGLE_PROJECT' => :google_project,
|
128
128
|
'GOOGLE_CLIENT_EMAIL' => :google_client_email,
|
129
129
|
'GOOGLE_JSON_KEY' => :google_json_key,
|
130
|
+
'GOOGLE_CDN_JSON_KEY' => :google_cdn_json_key,
|
131
|
+
'GOOGLE_CDN_LB' => :google_cdn_load_balancer,
|
132
|
+
'GOOGLE_CDN_SIGNURL_KEY' => :google_cdn_signurl_key,
|
133
|
+
'GOOGLE_CDN_SIGNURL_KEY_NAME' => :google_cdn_signurl_key_name,
|
134
|
+
'GCS_CDN_BUCKET_NAME' => :gcs_cdn_bucket_name,
|
130
135
|
'GCS_BUCKET_NAME' => :gcs_bucket_name,
|
131
136
|
'SMOKE_ONLY' => :smoke_only,
|
132
137
|
'NO_ADMIN' => :no_admin,
|
@@ -275,6 +280,14 @@ module Gitlab
|
|
275
280
|
end
|
276
281
|
end
|
277
282
|
|
283
|
+
def require_gcs_with_cdn_environment!
|
284
|
+
%w[GOOGLE_CDN_JSON_KEY GCS_CDN_BUCKET_NAME GOOGLE_CDN_LB GOOGLE_CDN_SIGNURL_KEY GOOGLE_CDN_SIGNURL_KEY_NAME].each do |env_key|
|
285
|
+
unless ENV.key?(env_key)
|
286
|
+
raise ArgumentError, "Environment variable #{env_key} must be set to run GCS with CDN enabled scenario"
|
287
|
+
end
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
278
291
|
def require_initial_password!
|
279
292
|
return unless env_var_value_if_defined('GITLAB_INITIAL_ROOT_PASSWORD').to_s.strip.empty?
|
280
293
|
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Gitlab
|
2
|
+
module QA
|
3
|
+
module Scenario
|
4
|
+
module Test
|
5
|
+
module Instance
|
6
|
+
class StagingRefGeo < DeploymentBase
|
7
|
+
def initialize
|
8
|
+
@suite = 'QA::EE::Scenario::Test::Geo'
|
9
|
+
end
|
10
|
+
|
11
|
+
def deployment_component
|
12
|
+
Component::StagingRef
|
13
|
+
end
|
14
|
+
|
15
|
+
def non_rspec_args
|
16
|
+
[
|
17
|
+
'--primary-address', deployment_component::ADDRESS,
|
18
|
+
'--secondary-address', deployment_component::GEO_SECONDARY_ADDRESS,
|
19
|
+
'--without-setup'
|
20
|
+
]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Scenario
|
6
|
+
module Test
|
7
|
+
module Integration
|
8
|
+
class RegistryWithCDN < Scenario::Template
|
9
|
+
def perform(release, *rspec_args)
|
10
|
+
Runtime::Env.require_gcs_with_cdn_environment!
|
11
|
+
|
12
|
+
Component::Gitlab.perform do |gitlab|
|
13
|
+
gitlab.release = release
|
14
|
+
gitlab.network = 'test'
|
15
|
+
gitlab.name = 'gitlab'
|
16
|
+
sign_url_key_path = gitlab.copy_key_file('GOOGLE_CDN_SIGNURL_KEY')
|
17
|
+
cdn_gcloud_path = gitlab.copy_key_file('GOOGLE_CDN_JSON_KEY')
|
18
|
+
|
19
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
20
|
+
external_url 'http://#{gitlab.name}.#{gitlab.network}';
|
21
|
+
registry_external_url 'http://#{gitlab.name}.#{gitlab.network}:5050';
|
22
|
+
|
23
|
+
registry['middleware'] = { 'storage' => [{ 'name' => 'googlecdn', 'options' => { 'baseurl' => '#{Runtime::Env.google_cdn_load_balancer}', 'privatekey' => '#{sign_url_key_path}', 'keyname' => '#{Runtime::Env.google_cdn_signurl_key_name}' } }] }
|
24
|
+
registry['storage'] = { 'gcs' => { 'bucket' => '#{Runtime::Env.gcs_cdn_bucket_name}', 'keyfile' => '#{cdn_gcloud_path}' } }
|
25
|
+
OMNIBUS
|
26
|
+
|
27
|
+
gitlab.instance do
|
28
|
+
Component::Specs.perform do |specs|
|
29
|
+
specs.suite = 'Test::Integration::RegistryWithCDN'
|
30
|
+
specs.release = gitlab.release
|
31
|
+
specs.network = gitlab.network
|
32
|
+
specs.args = [gitlab.address, *rspec_args]
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/lib/gitlab/qa/version.rb
CHANGED
data/lib/gitlab/qa.rb
CHANGED
@@ -43,6 +43,7 @@ module Gitlab
|
|
43
43
|
autoload :Release, 'gitlab/qa/scenario/test/instance/release'
|
44
44
|
autoload :Geo, 'gitlab/qa/scenario/test/instance/geo'
|
45
45
|
autoload :StagingGeo, 'gitlab/qa/scenario/test/instance/staging_geo'
|
46
|
+
autoload :StagingRefGeo, 'gitlab/qa/scenario/test/instance/staging_ref_geo'
|
46
47
|
autoload :Airgapped, 'gitlab/qa/scenario/test/instance/airgapped'
|
47
48
|
end
|
48
49
|
|
@@ -76,6 +77,7 @@ module Gitlab
|
|
76
77
|
autoload :RegistryTLS, 'gitlab/qa/scenario/test/integration/registry_tls'
|
77
78
|
autoload :ServicePingDisabled, 'gitlab/qa/scenario/test/integration/service_ping_disabled'
|
78
79
|
autoload :Integrations, 'gitlab/qa/scenario/test/integration/integrations'
|
80
|
+
autoload :RegistryWithCDN, 'gitlab/qa/scenario/test/integration/registry_with_cdn'
|
79
81
|
end
|
80
82
|
|
81
83
|
module Sanity
|