gitlab-qa 5.13.6 → 5.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +42 -1
- data/README.md +52 -5
- data/docs/run_qa_against_gdk.md +3 -0
- data/docs/what_tests_can_be_run.md +21 -6
- data/lib/gitlab/qa.rb +4 -1
- data/lib/gitlab/qa/component/base.rb +156 -0
- data/lib/gitlab/qa/component/elasticsearch.rb +5 -32
- data/lib/gitlab/qa/component/gitlab.rb +16 -88
- data/lib/gitlab/qa/component/internet_tunnel.rb +6 -29
- data/lib/gitlab/qa/component/jira.rb +5 -44
- data/lib/gitlab/qa/component/ldap.rb +8 -51
- data/lib/gitlab/qa/component/mail_hog.rb +5 -44
- data/lib/gitlab/qa/component/minio.rb +9 -34
- data/lib/gitlab/qa/component/postgresql.rb +4 -36
- data/lib/gitlab/qa/component/saml.rb +5 -54
- data/lib/gitlab/qa/component/specs.rb +10 -11
- data/lib/gitlab/qa/docker/command.rb +15 -3
- data/lib/gitlab/qa/docker/engine.rb +24 -1
- data/lib/gitlab/qa/docker/shellout.rb +2 -2
- data/lib/gitlab/qa/release.rb +33 -0
- data/lib/gitlab/qa/runner.rb +23 -15
- data/lib/gitlab/qa/runtime/env.rb +32 -1
- data/lib/gitlab/qa/runtime/scenario.rb +36 -0
- data/lib/gitlab/qa/scenario/test/instance/airgapped.rb +68 -0
- data/lib/gitlab/qa/scenario/test/integration/geo.rb +4 -2
- data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb +201 -0
- data/lib/gitlab/qa/scenario/test/integration/praefect.rb +18 -7
- data/lib/gitlab/qa/scenario/test/sanity/version.rb +1 -1
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +7 -4
- data/lib/gitlab/qa/scenario/test/integration/gitaly_ha.rb +0 -166
@@ -6,22 +6,9 @@ require 'securerandom'
|
|
6
6
|
module Gitlab
|
7
7
|
module QA
|
8
8
|
module Component
|
9
|
-
class SAML
|
10
|
-
|
11
|
-
|
12
|
-
SAML_IMAGE = 'jamedjo/test-saml-idp'.freeze
|
13
|
-
SAML_IMAGE_TAG = 'latest'.freeze
|
14
|
-
|
15
|
-
attr_reader :docker
|
16
|
-
attr_accessor :volumes, :network, :environment
|
17
|
-
attr_writer :name
|
18
|
-
|
19
|
-
def initialize
|
20
|
-
@docker = Docker::Engine.new
|
21
|
-
@environment = {}
|
22
|
-
@volumes = {}
|
23
|
-
@network_aliases = []
|
24
|
-
end
|
9
|
+
class SAML < Base
|
10
|
+
DOCKER_IMAGE = 'jamedjo/test-saml-idp'.freeze
|
11
|
+
DOCKER_IMAGE_TAG = 'latest'.freeze
|
25
12
|
|
26
13
|
def set_entity_id(entity_id)
|
27
14
|
@environment['SIMPLESAMLPHP_SP_ENTITY_ID'] = entity_id
|
@@ -31,18 +18,10 @@ module Gitlab
|
|
31
18
|
@environment['SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE'] = assertion_con_service
|
32
19
|
end
|
33
20
|
|
34
|
-
def add_network_alias(name)
|
35
|
-
@network_aliases.push(name)
|
36
|
-
end
|
37
|
-
|
38
21
|
def name
|
39
22
|
@name ||= "saml-qa-idp"
|
40
23
|
end
|
41
24
|
|
42
|
-
def hostname
|
43
|
-
"#{name}.#{network}"
|
44
|
-
end
|
45
|
-
|
46
25
|
def group_name
|
47
26
|
@group_name ||= "saml_sso_group-#{SecureRandom.hex(4)}"
|
48
27
|
end
|
@@ -50,25 +29,12 @@ module Gitlab
|
|
50
29
|
def instance
|
51
30
|
raise 'Please provide a block!' unless block_given?
|
52
31
|
|
53
|
-
|
54
|
-
start
|
55
|
-
|
56
|
-
yield self
|
57
|
-
ensure
|
58
|
-
teardown
|
59
|
-
end
|
60
|
-
|
61
|
-
def prepare
|
62
|
-
pull
|
63
|
-
|
64
|
-
return if @docker.network_exists?(network)
|
65
|
-
|
66
|
-
@docker.network_create(network)
|
32
|
+
super
|
67
33
|
end
|
68
34
|
|
69
35
|
# rubocop:disable Metrics/AbcSize
|
70
36
|
def start
|
71
|
-
docker.run(
|
37
|
+
docker.run(image, tag) do |command|
|
72
38
|
command << '-d '
|
73
39
|
command << "--name #{name}"
|
74
40
|
command << "--net #{network}"
|
@@ -91,21 +57,6 @@ module Gitlab
|
|
91
57
|
end
|
92
58
|
# rubocop:enable Metrics/AbcSize
|
93
59
|
|
94
|
-
def restart
|
95
|
-
@docker.restart(name)
|
96
|
-
end
|
97
|
-
|
98
|
-
def teardown
|
99
|
-
raise 'Invalid instance name!' unless name
|
100
|
-
|
101
|
-
@docker.stop(name)
|
102
|
-
@docker.remove(name)
|
103
|
-
end
|
104
|
-
|
105
|
-
def pull
|
106
|
-
@docker.pull(SAML_IMAGE, SAML_IMAGE_TAG)
|
107
|
-
end
|
108
|
-
|
109
60
|
def set_sandbox_name(sandbox_name)
|
110
61
|
::Gitlab::QA::Runtime::Env.gitlab_sandbox_name = sandbox_name
|
111
62
|
end
|
@@ -8,7 +8,7 @@ module Gitlab
|
|
8
8
|
# the `qa/` directory located in GitLab CE / EE repositories.
|
9
9
|
#
|
10
10
|
class Specs < Scenario::Template
|
11
|
-
attr_accessor :suite, :release, :network, :args, :volumes, :env
|
11
|
+
attr_accessor :suite, :release, :network, :args, :volumes, :env, :runner_network
|
12
12
|
|
13
13
|
def initialize
|
14
14
|
@docker = Docker::Engine.new
|
@@ -17,18 +17,11 @@ module Gitlab
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def perform # rubocop:disable Metrics/AbcSize
|
20
|
+
return puts "Skipping tests." if skip_tests?
|
21
|
+
|
20
22
|
raise ArgumentError unless [suite, release].all?
|
21
23
|
|
22
|
-
if release.
|
23
|
-
Docker::Command.execute(
|
24
|
-
[
|
25
|
-
'login',
|
26
|
-
'--username gitlab-qa-bot',
|
27
|
-
%(--password "#{Runtime::Env.dev_access_token_variable}"),
|
28
|
-
QA::Release::DEV_REGISTRY
|
29
|
-
]
|
30
|
-
)
|
31
|
-
end
|
24
|
+
@docker.login(**release.login_params) if release.login_params
|
32
25
|
|
33
26
|
puts "Running test suite `#{suite}` for #{release.project_name}"
|
34
27
|
|
@@ -51,6 +44,12 @@ module Gitlab
|
|
51
44
|
command.name(name)
|
52
45
|
end
|
53
46
|
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def skip_tests?
|
51
|
+
Runtime::Scenario.attributes.include?(:run_tests) && !Runtime::Scenario.run_tests
|
52
|
+
end
|
54
53
|
end
|
55
54
|
end
|
56
55
|
end
|
@@ -4,8 +4,9 @@ module Gitlab
|
|
4
4
|
class Command
|
5
5
|
attr_reader :args
|
6
6
|
|
7
|
-
def initialize(cmd = nil)
|
7
|
+
def initialize(cmd = nil, mask_secrets: nil)
|
8
8
|
@args = Array(cmd)
|
9
|
+
@mask_secrets = Array(mask_secrets)
|
9
10
|
end
|
10
11
|
|
11
12
|
def <<(*args)
|
@@ -28,6 +29,17 @@ module Gitlab
|
|
28
29
|
"docker #{@args.join(' ')}"
|
29
30
|
end
|
30
31
|
|
32
|
+
# Returns a masked string form of a Command
|
33
|
+
#
|
34
|
+
# @example
|
35
|
+
# Command.new('a docker command', mask_secrets: 'command').mask_secrets #=> 'a docker *****'
|
36
|
+
# Command.new('a docker command', mask_secrets: %w[docker command]).mask_secrets #=> 'a ***** *****'
|
37
|
+
#
|
38
|
+
# @return [String] The masked command string
|
39
|
+
def mask_secrets
|
40
|
+
@mask_secrets.each_with_object(to_s) { |secret, s| s.gsub!(secret, '*****') }
|
41
|
+
end
|
42
|
+
|
31
43
|
def ==(other)
|
32
44
|
to_s == other.to_s
|
33
45
|
end
|
@@ -36,8 +48,8 @@ module Gitlab
|
|
36
48
|
Docker::Shellout.new(self).execute!(&block)
|
37
49
|
end
|
38
50
|
|
39
|
-
def self.execute(cmd, &block)
|
40
|
-
new(cmd).execute!(&block)
|
51
|
+
def self.execute(cmd, mask_secrets: nil, &block)
|
52
|
+
new(cmd, mask_secrets: mask_secrets).execute!(&block)
|
41
53
|
end
|
42
54
|
end
|
43
55
|
end
|
@@ -3,11 +3,16 @@ module Gitlab
|
|
3
3
|
module Docker
|
4
4
|
class Engine
|
5
5
|
DOCKER_HOST = ENV['DOCKER_HOST'] || 'http://localhost'
|
6
|
+
PRIVILEGED_COMMANDS = [/^iptables.*/].freeze
|
6
7
|
|
7
8
|
def hostname
|
8
9
|
URI(DOCKER_HOST).host
|
9
10
|
end
|
10
11
|
|
12
|
+
def login(username:, password:, registry:)
|
13
|
+
Docker::Command.execute(%(login --username "#{username}" --password "#{password}" #{registry}), mask_secrets: password)
|
14
|
+
end
|
15
|
+
|
11
16
|
def pull(image, tag)
|
12
17
|
Docker::Command.execute("pull #{image}:#{tag}")
|
13
18
|
end
|
@@ -23,8 +28,18 @@ module Gitlab
|
|
23
28
|
end
|
24
29
|
end
|
25
30
|
|
31
|
+
def privileged_command?(command)
|
32
|
+
PRIVILEGED_COMMANDS.each do |privileged_regex|
|
33
|
+
return true if command.match(privileged_regex)
|
34
|
+
end
|
35
|
+
|
36
|
+
false
|
37
|
+
end
|
38
|
+
|
26
39
|
def exec(name, command)
|
27
|
-
|
40
|
+
cmd = ['exec']
|
41
|
+
cmd << '--privileged' if privileged_command?(command)
|
42
|
+
Docker::Command.execute("#{cmd.join(' ')} #{name} bash -c '#{command}'")
|
28
43
|
end
|
29
44
|
|
30
45
|
def read_file(image, tag, path, &block)
|
@@ -63,6 +78,14 @@ module Gitlab
|
|
63
78
|
def port(name, port)
|
64
79
|
Docker::Command.execute("port #{name} #{port}/tcp")
|
65
80
|
end
|
81
|
+
|
82
|
+
def running?(name)
|
83
|
+
Docker::Command.execute("ps -f name=#{name}").include?(name)
|
84
|
+
end
|
85
|
+
|
86
|
+
def ps(name = nil)
|
87
|
+
Docker::Command.execute(['ps', name].compact.join(' '))
|
88
|
+
end
|
66
89
|
end
|
67
90
|
end
|
68
91
|
end
|
@@ -10,7 +10,7 @@ module Gitlab
|
|
10
10
|
@command = command
|
11
11
|
@output = []
|
12
12
|
|
13
|
-
puts "Docker shell command: `#{@command}`"
|
13
|
+
puts "Docker shell command: `#{@command.mask_secrets}`"
|
14
14
|
end
|
15
15
|
|
16
16
|
def execute!
|
@@ -28,7 +28,7 @@ module Gitlab
|
|
28
28
|
end
|
29
29
|
|
30
30
|
if wait.value.exited? && wait.value.exitstatus.nonzero?
|
31
|
-
raise StatusError, "Docker command `#{@command}` failed!"
|
31
|
+
raise StatusError, "Docker command `#{@command.mask_secrets}` failed!"
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
data/lib/gitlab/qa/release.rb
CHANGED
@@ -135,6 +135,35 @@ module Gitlab
|
|
135
135
|
end
|
136
136
|
end
|
137
137
|
|
138
|
+
def login_params
|
139
|
+
return if Runtime::Env.skip_pull?
|
140
|
+
|
141
|
+
if dev_gitlab_org?
|
142
|
+
Runtime::Env.require_qa_dev_access_token!
|
143
|
+
|
144
|
+
{
|
145
|
+
username: Runtime::Env.gitlab_dev_username,
|
146
|
+
password: Runtime::Env.dev_access_token_variable,
|
147
|
+
registry: DEV_REGISTRY
|
148
|
+
}
|
149
|
+
elsif omnibus_mirror?
|
150
|
+
username, password = if Runtime::Env.ci_job_token && Runtime::Env.ci_pipeline_source == 'pipeline'
|
151
|
+
['gitlab-ci-token', Runtime::Env.ci_job_token]
|
152
|
+
elsif Runtime::Env.qa_container_registry_access_token
|
153
|
+
[Runtime::Env.gitlab_username, Runtime::Env.qa_container_registry_access_token]
|
154
|
+
else
|
155
|
+
Runtime::Env.require_qa_access_token!
|
156
|
+
|
157
|
+
[Runtime::Env.gitlab_username, Runtime::Env.qa_access_token]
|
158
|
+
end
|
159
|
+
{
|
160
|
+
username: username,
|
161
|
+
password: password,
|
162
|
+
registry: COM_REGISTRY
|
163
|
+
}
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
138
167
|
def dev_gitlab_org?
|
139
168
|
image.start_with?(DEV_REGISTRY)
|
140
169
|
end
|
@@ -147,6 +176,10 @@ module Gitlab
|
|
147
176
|
canonical? || release.match?(CUSTOM_GITLAB_IMAGE_REGEX)
|
148
177
|
end
|
149
178
|
|
179
|
+
def api_project_name
|
180
|
+
project_name.gsub('ce', 'foss').gsub('-ee', '')
|
181
|
+
end
|
182
|
+
|
150
183
|
private
|
151
184
|
|
152
185
|
def canonical?
|
data/lib/gitlab/qa/runner.rb
CHANGED
@@ -4,22 +4,20 @@ module Gitlab
|
|
4
4
|
module QA
|
5
5
|
# rubocop:disable Metrics/AbcSize
|
6
6
|
class Runner
|
7
|
-
# These options are implemented in the QA framework (i.e., in the CE/EE codebase)
|
8
|
-
# They're included here so that gitlab-qa treats them as valid options
|
9
|
-
PASS_THROUGH_OPTS = [
|
10
|
-
['--address URL', 'Address of the instance to test'],
|
11
|
-
['--enable-feature FEATURE_FLAG', 'Enable a feature before running tests'],
|
12
|
-
['--mattermost-address URL', 'Address of the Mattermost server'],
|
13
|
-
['--parallel', 'Execute tests in parallel'],
|
14
|
-
['--loop', 'Execute tests in a loop']
|
15
|
-
].freeze
|
16
|
-
|
17
7
|
def self.run(args)
|
18
|
-
|
8
|
+
Runtime::Scenario.define(:teardown, true)
|
9
|
+
Runtime::Scenario.define(:run_tests, true)
|
10
|
+
|
11
|
+
@options = OptionParser.new do |opts|
|
19
12
|
opts.banner = 'Usage: gitlab-qa [options] Scenario URL [[--] path] [rspec_options]'
|
20
13
|
|
21
|
-
|
22
|
-
|
14
|
+
opts.on('--no-teardown', 'Skip teardown of containers after the scenario completes.') do
|
15
|
+
Runtime::Scenario.define(:teardown, false)
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on('--no-tests', 'Orchestrates the docker containers but does not run the tests. Implies --no-teardown') do
|
19
|
+
Runtime::Scenario.define(:run_tests, false)
|
20
|
+
Runtime::Scenario.define(:teardown, false)
|
23
21
|
end
|
24
22
|
|
25
23
|
opts.on_tail('-v', '--version', 'Show the version') do
|
@@ -33,19 +31,29 @@ module Gitlab
|
|
33
31
|
exit
|
34
32
|
end
|
35
33
|
|
36
|
-
|
34
|
+
begin
|
35
|
+
opts.parse(args)
|
36
|
+
rescue OptionParser::InvalidOption
|
37
|
+
# Ignore invalid options and options that are passed through to the tests
|
38
|
+
end
|
37
39
|
end
|
38
40
|
|
41
|
+
args.reject! { |arg| gitlab_qa_options.include?(arg) }
|
42
|
+
|
39
43
|
if args.size >= 1
|
40
44
|
Scenario
|
41
45
|
.const_get(args.shift)
|
42
46
|
.perform(*args)
|
43
47
|
else
|
44
|
-
puts options
|
48
|
+
puts @options
|
45
49
|
exit 1
|
46
50
|
end
|
47
51
|
end
|
48
52
|
# rubocop:enable Metrics/AbcSize
|
53
|
+
|
54
|
+
def self.gitlab_qa_options
|
55
|
+
@gitlab_qa_options ||= @options.top.list.map(&:long).flatten
|
56
|
+
end
|
49
57
|
end
|
50
58
|
end
|
51
59
|
end
|
@@ -38,6 +38,7 @@ module Gitlab
|
|
38
38
|
'SIGNUP_DISABLED' => :signup_disabled,
|
39
39
|
'QA_ADDITIONAL_REPOSITORY_STORAGE' => :qa_additional_repository_storage,
|
40
40
|
'QA_PRAEFECT_REPOSITORY_STORAGE' => :qa_praefect_repository_storage,
|
41
|
+
'QA_GITALY_NON_CLUSTER_STORAGE' => :qa_gitaly_non_cluster_storage,
|
41
42
|
'QA_COOKIES' => :qa_cookie,
|
42
43
|
'QA_DEBUG' => :qa_debug,
|
43
44
|
'QA_LOG_PATH' => :qa_log_path,
|
@@ -84,18 +85,38 @@ module Gitlab
|
|
84
85
|
send(:attr_accessor, accessor) # rubocop:disable GitlabSecurity/PublicSend
|
85
86
|
end
|
86
87
|
|
88
|
+
def gitlab_username
|
89
|
+
ENV['GITLAB_USERNAME'] || 'gitlab-qa'
|
90
|
+
end
|
91
|
+
|
92
|
+
def gitlab_dev_username
|
93
|
+
ENV['GITLAB_DEV_USERNAME'] || 'gitlab-qa-bot'
|
94
|
+
end
|
95
|
+
|
87
96
|
def gitlab_api_base
|
88
97
|
ENV['GITLAB_API_BASE'] || 'https://gitlab.com/api/v4'
|
89
98
|
end
|
90
99
|
|
100
|
+
def gitlab_bot_multi_project_pipeline_polling_token
|
101
|
+
ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
|
102
|
+
end
|
103
|
+
|
91
104
|
def ci_job_name
|
92
105
|
ENV['CI_JOB_NAME']
|
93
106
|
end
|
94
107
|
|
108
|
+
def ci_job_token
|
109
|
+
ENV['CI_JOB_TOKEN']
|
110
|
+
end
|
111
|
+
|
95
112
|
def ci_job_url
|
96
113
|
ENV['CI_JOB_URL']
|
97
114
|
end
|
98
115
|
|
116
|
+
def ci_pipeline_source
|
117
|
+
ENV['CI_PIPELINE_SOURCE']
|
118
|
+
end
|
119
|
+
|
99
120
|
def ci_project_name
|
100
121
|
ENV['CI_PROJECT_NAME']
|
101
122
|
end
|
@@ -132,6 +153,10 @@ module Gitlab
|
|
132
153
|
ENV['GITLAB_QA_DEV_ACCESS_TOKEN']
|
133
154
|
end
|
134
155
|
|
156
|
+
def qa_container_registry_access_token
|
157
|
+
ENV['GITLAB_QA_CONTAINER_REGISTRY_ACCESS_TOKEN']
|
158
|
+
end
|
159
|
+
|
135
160
|
def host_artifacts_dir
|
136
161
|
@host_artifacts_dir ||= File.join(ENV['QA_ARTIFACTS_DIR'] || '/tmp/gitlab-qa', Runtime::Env.run_id)
|
137
162
|
end
|
@@ -196,7 +221,7 @@ module Gitlab
|
|
196
221
|
end
|
197
222
|
|
198
223
|
def skip_pull?
|
199
|
-
(ENV['QA_SKIP_PULL']
|
224
|
+
enabled?(ENV['QA_SKIP_PULL'], default: false)
|
200
225
|
end
|
201
226
|
|
202
227
|
def gitlab_qa_formless_login_token
|
@@ -205,6 +230,12 @@ module Gitlab
|
|
205
230
|
|
206
231
|
private
|
207
232
|
|
233
|
+
def enabled?(value, default: true)
|
234
|
+
return default if value.nil?
|
235
|
+
|
236
|
+
(value =~ /^(false|no|0)$/i) != 0
|
237
|
+
end
|
238
|
+
|
208
239
|
def env_value_if_defined(variable)
|
209
240
|
# Pass through the variables if they are defined in the environment
|
210
241
|
return "$#{variable}" if ENV[variable]
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Runtime
|
6
|
+
##
|
7
|
+
# Singleton approach to global test scenario arguments.
|
8
|
+
#
|
9
|
+
module Scenario
|
10
|
+
extend self
|
11
|
+
|
12
|
+
def attributes
|
13
|
+
@attributes ||= {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def define(attribute, value)
|
17
|
+
attributes.store(attribute.to_sym, value)
|
18
|
+
|
19
|
+
define_singleton_method(attribute) do
|
20
|
+
attributes[attribute.to_sym].tap do |value|
|
21
|
+
if value.to_s.empty?
|
22
|
+
raise ArgumentError, "Empty `#{attribute}` attribute!"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# rubocop:disable Style/MethodMissing
|
29
|
+
def method_missing(name, *)
|
30
|
+
raise ArgumentError, "Scenario attribute `#{name}` not defined!"
|
31
|
+
end
|
32
|
+
# rubocop:enable Style/MethodMissing
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|