gitlab-qa 6.23.0 → 7.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +31 -30
- data/docs/configuring_omnibus.md +208 -0
- data/docs/what_tests_can_be_run.md +12 -8
- data/gitlab-qa.gemspec +6 -4
- data/lib/gitlab/qa.rb +7 -0
- data/lib/gitlab/qa/component/base.rb +2 -2
- data/lib/gitlab/qa/component/elasticsearch.rb +1 -1
- data/lib/gitlab/qa/component/gitlab.rb +21 -24
- data/lib/gitlab/qa/component/internet_tunnel.rb +4 -1
- data/lib/gitlab/qa/component/jira.rb +1 -1
- data/lib/gitlab/qa/component/ldap.rb +1 -1
- data/lib/gitlab/qa/component/mail_hog.rb +1 -1
- data/lib/gitlab/qa/component/minio.rb +3 -11
- data/lib/gitlab/qa/component/postgresql.rb +1 -1
- data/lib/gitlab/qa/component/saml.rb +1 -1
- data/lib/gitlab/qa/component/specs.rb +2 -2
- data/lib/gitlab/qa/docker/engine.rb +28 -3
- data/lib/gitlab/qa/runner.rb +52 -5
- data/lib/gitlab/qa/runtime/env.rb +1 -0
- data/lib/gitlab/qa/runtime/omnibus_configuration.rb +80 -0
- data/lib/gitlab/qa/runtime/omnibus_configurations/default.rb +25 -0
- data/lib/gitlab/qa/runtime/omnibus_configurations/object_storage.rb +48 -0
- data/lib/gitlab/qa/runtime/omnibus_configurations/packages.rb +17 -0
- data/lib/gitlab/qa/scenario/cli_commands.rb +3 -3
- data/lib/gitlab/qa/scenario/test/instance/relative_url.rb +1 -3
- data/lib/gitlab/qa/scenario/test/instance/repository_storage.rb +1 -1
- data/lib/gitlab/qa/scenario/test/integration/actioncable.rb +1 -3
- data/lib/gitlab/qa/scenario/test/integration/geo.rb +4 -5
- data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb +4 -5
- data/lib/gitlab/qa/scenario/test/integration/group_saml.rb +1 -1
- data/lib/gitlab/qa/scenario/test/integration/instance_saml.rb +1 -1
- data/lib/gitlab/qa/scenario/test/integration/kubernetes.rb +1 -1
- data/lib/gitlab/qa/scenario/test/integration/ldap.rb +1 -4
- data/lib/gitlab/qa/scenario/test/integration/ldap_no_server.rb +1 -1
- data/lib/gitlab/qa/scenario/test/integration/ldap_no_tls.rb +1 -1
- data/lib/gitlab/qa/scenario/test/integration/ldap_tls.rb +1 -1
- data/lib/gitlab/qa/scenario/test/integration/mattermost.rb +1 -1
- data/lib/gitlab/qa/scenario/test/integration/mtls.rb +2 -2
- data/lib/gitlab/qa/scenario/test/integration/smtp.rb +1 -1
- data/lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb +1 -1
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +12 -9
- data/lib/gitlab/qa/scenario/test/integration/object_storage.rb +0 -64
- data/lib/gitlab/qa/scenario/test/integration/packages.rb +0 -36
@@ -65,6 +65,7 @@ module Gitlab
|
|
65
65
|
'KNAPSACK_TEST_FILE_PATTERN' => :knapsack_test_file_pattern,
|
66
66
|
'KNAPSACK_TEST_DIR' => :knapsack_test_dir,
|
67
67
|
'CI' => :ci,
|
68
|
+
'CI_JOB_NAME' => :ci_job_name,
|
68
69
|
'CI_JOB_URL' => :ci_job_url,
|
69
70
|
'CI_RUNNER_ID' => :ci_runner_id,
|
70
71
|
'CI_SERVER_HOST' => :ci_server_host,
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
module QA
|
7
|
+
module Runtime
|
8
|
+
class OmnibusConfiguration
|
9
|
+
# @param prefixed_config The configuration to be prefixed to the new configuration
|
10
|
+
def initialize(prefixed_config = nil)
|
11
|
+
@config = ["# Generated by GitLab QA Omnibus Configurator at #{Time.now.strftime('%Y-%m-%d %H:%M:%S')}"]
|
12
|
+
|
13
|
+
return unless prefixed_config
|
14
|
+
|
15
|
+
# remove generation statement if it exists within the prefixed configuration that was passed
|
16
|
+
# and insert the rest AFTER the very first generation statement
|
17
|
+
config_to_insert = prefixed_config.to_s
|
18
|
+
generation_regexp = /# Generated by GitLab QA Omnibus Configurator.*\n/
|
19
|
+
config_to_insert = config_to_insert.gsub(generation_regexp, '') if config_to_insert.match?(generation_regexp)
|
20
|
+
@config.insert(1, config_to_insert)
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_s
|
24
|
+
sanitize!.join("\n")
|
25
|
+
end
|
26
|
+
|
27
|
+
def configuration
|
28
|
+
raise NotImplementedError
|
29
|
+
end
|
30
|
+
|
31
|
+
# Before hook for any additional configuration
|
32
|
+
# This would usually be a container that needs to be running
|
33
|
+
# @return Any instance of [Gitlab::QA::Component::Base]
|
34
|
+
def prepare; end
|
35
|
+
|
36
|
+
# Commands to execute before GitLab boots
|
37
|
+
def exec_commands
|
38
|
+
[]
|
39
|
+
end
|
40
|
+
|
41
|
+
# Ensures no duplicate entries and sanitizes configurations
|
42
|
+
# @raise RuntimeError if competing configurations exist
|
43
|
+
# rubocop:disable Metrics/AbcSize
|
44
|
+
def sanitize!
|
45
|
+
sanitized = @config.map do |config|
|
46
|
+
next config if config.start_with?('#') || config.match(/\w+\(/) # allow for comments and method invocations
|
47
|
+
|
48
|
+
# sometimes "=" is part of a Hash. Only split based on the first "="
|
49
|
+
k, v = config.split("=", 2)
|
50
|
+
# make sure each config is well-formed
|
51
|
+
# e.g., gitlab_rails['packages_enabled'] = true
|
52
|
+
# NOT gitlab_rails['packages_enabled']=true
|
53
|
+
|
54
|
+
v.nil? ? k.strip : "#{k.strip} = #{v.strip.tr('"', "'")}".strip
|
55
|
+
end.uniq
|
56
|
+
|
57
|
+
errors = []
|
58
|
+
|
59
|
+
# check for duplicates
|
60
|
+
duplicate_keys = []
|
61
|
+
duplicates = sanitized.reject do |n|
|
62
|
+
key = n.split('=').first
|
63
|
+
duplicate_keys << key unless duplicate_keys.include?(key)
|
64
|
+
end
|
65
|
+
|
66
|
+
duplicates.each { |duplicate| errors << "Duplicate entry found: `#{duplicate}`" }
|
67
|
+
|
68
|
+
raise "Errors exist within the Omnibus Configuration!\n#{errors.join(',')}" if errors.any?
|
69
|
+
|
70
|
+
@config = sanitized
|
71
|
+
end
|
72
|
+
# rubocop:enable Metrics/AbcSize
|
73
|
+
|
74
|
+
def <<(config)
|
75
|
+
@config << config.strip unless config.strip.empty?
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Runtime
|
6
|
+
module OmnibusConfigurations
|
7
|
+
# Default Configuration for Omnibus
|
8
|
+
# All runs will include this configuration
|
9
|
+
class Default < Runtime::OmnibusConfiguration
|
10
|
+
def configuration
|
11
|
+
<<~OMNIBUS
|
12
|
+
gitlab_rails['gitlab_default_theme'] = 10 # Light Red Theme
|
13
|
+
gitlab_rails['gitlab_disable_animations'] = true # Disable animations
|
14
|
+
gitlab_rails['application_settings_cache_seconds'] = 0 # Settings cache expiry
|
15
|
+
OMNIBUS
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.configuration
|
19
|
+
new.configuration
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Runtime
|
6
|
+
module OmnibusConfigurations
|
7
|
+
class ObjectStorage < Default
|
8
|
+
TYPES = %w[artifacts external_diffs lfs uploads packages dependency_proxy].freeze
|
9
|
+
|
10
|
+
def configuration
|
11
|
+
TYPES.each_with_object(+'') do |object_type, omnibus|
|
12
|
+
omnibus << <<~OMNIBUS
|
13
|
+
gitlab_rails['#{object_type}_enabled'] = true
|
14
|
+
gitlab_rails['#{object_type}_storage_path'] = '/var/opt/gitlab/gitlab-rails/shared/#{object_type}'
|
15
|
+
gitlab_rails['#{object_type}_object_store_enabled'] = true
|
16
|
+
gitlab_rails['#{object_type}_object_store_remote_directory'] = '#{object_type}-bucket'
|
17
|
+
gitlab_rails['#{object_type}_object_store_background_upload'] = false
|
18
|
+
gitlab_rails['#{object_type}_object_store_direct_upload'] = true
|
19
|
+
gitlab_rails['#{object_type}_object_store_proxy_download'] = true
|
20
|
+
gitlab_rails['#{object_type}_object_store_connection'] = #{minio.to_config}
|
21
|
+
OMNIBUS
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def prepare
|
26
|
+
minio.network = 'test'
|
27
|
+
|
28
|
+
TYPES.each do |bucket_name|
|
29
|
+
minio.add_bucket("#{bucket_name}-bucket")
|
30
|
+
end
|
31
|
+
|
32
|
+
minio
|
33
|
+
end
|
34
|
+
|
35
|
+
def exec_commands
|
36
|
+
QA::Scenario::CLICommands.git_lfs_install_commands
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def minio
|
42
|
+
@minio ||= Component::Minio.new
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Runtime
|
6
|
+
module OmnibusConfigurations
|
7
|
+
class Packages < Default
|
8
|
+
def configuration
|
9
|
+
<<~OMNIBUS
|
10
|
+
gitlab_rails['packages_enabled'] = true
|
11
|
+
OMNIBUS
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -4,11 +4,11 @@ module Gitlab
|
|
4
4
|
module CLICommands
|
5
5
|
GIT_LFS_VERSION = '2.8.0'.freeze
|
6
6
|
|
7
|
-
def git_lfs_install_commands
|
8
|
-
|
7
|
+
def self.git_lfs_install_commands
|
8
|
+
[
|
9
9
|
"cd /tmp ; curl -qsL https://github.com/git-lfs/git-lfs/releases/download/v#{GIT_LFS_VERSION}/git-lfs-linux-amd64-v#{GIT_LFS_VERSION}.tar.gz | tar xzvf -",
|
10
10
|
'cp /tmp/git-lfs /usr/local/bin'
|
11
|
-
]
|
11
|
+
].freeze
|
12
12
|
end
|
13
13
|
end
|
14
14
|
end
|
@@ -10,9 +10,7 @@ module Gitlab
|
|
10
10
|
gitlab.network = 'test'
|
11
11
|
gitlab.relative_path = '/relative'
|
12
12
|
|
13
|
-
gitlab.
|
14
|
-
external_url '#{gitlab.address}';
|
15
|
-
OMNIBUS
|
13
|
+
gitlab.omnibus_configuration << "external_url '#{gitlab.address}'"
|
16
14
|
|
17
15
|
gitlab.instance do
|
18
16
|
Component::Specs.perform do |specs|
|
@@ -9,7 +9,7 @@ module Gitlab
|
|
9
9
|
gitlab.release = QA::Release.new(release)
|
10
10
|
gitlab.name = 'gitlab'
|
11
11
|
gitlab.network = 'test'
|
12
|
-
gitlab.
|
12
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
13
13
|
git_data_dirs({
|
14
14
|
'default' => {
|
15
15
|
'path' => '/var/opt/gitlab/git-data/repositories/default'
|
@@ -9,9 +9,7 @@ module Gitlab
|
|
9
9
|
gitlab.release = QA::Release.new(release)
|
10
10
|
gitlab.name = 'gitlab-actioncable'
|
11
11
|
gitlab.network = 'test'
|
12
|
-
gitlab.
|
13
|
-
actioncable['enable'] = true;
|
14
|
-
OMNIBUS
|
12
|
+
gitlab.omnibus_configuration << "actioncable['enable'] = true"
|
15
13
|
|
16
14
|
gitlab.instance do
|
17
15
|
puts "Running actioncable specs!"
|
@@ -4,7 +4,6 @@ module Gitlab
|
|
4
4
|
module Test
|
5
5
|
module Integration
|
6
6
|
class Geo < Scenario::Template
|
7
|
-
include Scenario::CLICommands
|
8
7
|
##
|
9
8
|
# rubocop:disable Lint/MissingCopEnableDirective
|
10
9
|
# rubocop:disable Metrics/AbcSize
|
@@ -20,7 +19,7 @@ module Gitlab
|
|
20
19
|
primary.release = release
|
21
20
|
primary.name = 'gitlab-primary'
|
22
21
|
primary.network = 'geo'
|
23
|
-
primary.
|
22
|
+
primary.omnibus_configuration << <<~OMNIBUS
|
24
23
|
gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc';
|
25
24
|
geo_primary_role['enable'] = true;
|
26
25
|
gitlab_rails['db_password'] = 'mypass';
|
@@ -36,14 +35,14 @@ module Gitlab
|
|
36
35
|
sidekiq['concurrency'] = 2;
|
37
36
|
puma['worker_processes'] = 2;
|
38
37
|
OMNIBUS
|
39
|
-
primary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
|
38
|
+
primary.exec_commands = fast_ssh_key_lookup_commands + QA::Scenario::CLICommands.git_lfs_install_commands
|
40
39
|
|
41
40
|
primary.instance do
|
42
41
|
Component::Gitlab.perform do |secondary|
|
43
42
|
secondary.release = release
|
44
43
|
secondary.name = 'gitlab-secondary'
|
45
44
|
secondary.network = 'geo'
|
46
|
-
secondary.
|
45
|
+
secondary.omnibus_configuration << <<~OMNIBUS
|
47
46
|
geo_secondary['db_fdw'] = true;
|
48
47
|
geo_secondary_role['enable'] = true;
|
49
48
|
gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc';
|
@@ -58,7 +57,7 @@ module Gitlab
|
|
58
57
|
sidekiq['concurrency'] = 2;
|
59
58
|
puma['worker_processes'] = 2;
|
60
59
|
OMNIBUS
|
61
|
-
secondary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
|
60
|
+
secondary.exec_commands = fast_ssh_key_lookup_commands + QA::Scenario::CLICommands.git_lfs_install_commands
|
62
61
|
|
63
62
|
secondary.act do
|
64
63
|
# TODO, we do not wait for secondary to start because of
|
@@ -39,7 +39,7 @@ module Gitlab
|
|
39
39
|
praefect.network = @network
|
40
40
|
praefect.skip_availability_check = true
|
41
41
|
|
42
|
-
praefect.
|
42
|
+
praefect.omnibus_configuration << praefect_omnibus_configuration
|
43
43
|
|
44
44
|
praefect.instance(skip_teardown: true)
|
45
45
|
end
|
@@ -49,7 +49,7 @@ module Gitlab
|
|
49
49
|
gitlab.name = gitlab_name
|
50
50
|
gitlab.network = @network
|
51
51
|
|
52
|
-
gitlab.
|
52
|
+
gitlab.omnibus_configuration << gitlab_omnibus_configuration
|
53
53
|
gitlab.instance do
|
54
54
|
puts "Running Gitaly Cluster specs!"
|
55
55
|
|
@@ -131,8 +131,7 @@ module Gitlab
|
|
131
131
|
|
132
132
|
def gitaly_omnibus_configuration
|
133
133
|
<<~OMNIBUS
|
134
|
-
#{disable_other_services}
|
135
|
-
prometheus['enable'] = true;
|
134
|
+
#{disable_other_services.sub(/(prometheus\['enable'\]) = false/, '\1 = true')}
|
136
135
|
prometheus_monitoring['enable'] = false;
|
137
136
|
gitaly['enable'] = true;
|
138
137
|
gitaly['listen_addr'] = '0.0.0.0:8075';
|
@@ -197,7 +196,7 @@ module Gitlab
|
|
197
196
|
gitaly.name = name
|
198
197
|
gitaly.network = @network
|
199
198
|
gitaly.skip_availability_check = true
|
200
|
-
gitaly.
|
199
|
+
gitaly.omnibus_configuration << gitaly_omnibus_configuration
|
201
200
|
gitaly.instance(skip_teardown: true)
|
202
201
|
end
|
203
202
|
end
|
@@ -17,7 +17,7 @@ module Gitlab
|
|
17
17
|
saml.set_assertion_consumer_service("#{gitlab.address}/users/auth/saml/callback")
|
18
18
|
saml.set_simple_saml_hostname
|
19
19
|
|
20
|
-
gitlab.
|
20
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
21
21
|
gitlab_rails['omniauth_enabled'] = true;
|
22
22
|
gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'];
|
23
23
|
gitlab_rails['omniauth_block_auto_created_users'] = false;
|
@@ -21,7 +21,7 @@ module Gitlab
|
|
21
21
|
tunnel_registry.gitlab_hostname = gitlab.hostname
|
22
22
|
tunnel_registry.network = 'test'
|
23
23
|
|
24
|
-
gitlab.
|
24
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
25
25
|
external_url '#{tunnel_gitlab.url}';
|
26
26
|
nginx['listen_port'] = 80;
|
27
27
|
nginx['listen_https'] = false;
|
@@ -27,7 +27,7 @@ module Gitlab
|
|
27
27
|
end
|
28
28
|
|
29
29
|
def ldap_servers_omnibus_config
|
30
|
-
|
30
|
+
YAML.safe_load <<~CFG
|
31
31
|
main:
|
32
32
|
label: LDAP
|
33
33
|
host: #{ldap_hostname}
|
@@ -44,9 +44,6 @@ module Gitlab
|
|
44
44
|
external_groups: ''
|
45
45
|
sync_ssh_keys: false
|
46
46
|
CFG
|
47
|
-
|
48
|
-
# Quotes get eaten up when the string is set in the environment
|
49
|
-
config.to_s.gsub("\"", "\\\"")
|
50
47
|
end
|
51
48
|
|
52
49
|
def ldap_hostname
|
@@ -12,7 +12,7 @@ module Gitlab
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def configure_omnibus(gitlab)
|
15
|
-
gitlab.
|
15
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
16
16
|
gitlab_rails['ldap_enabled'] = true;
|
17
17
|
gitlab_rails['ldap_servers'] = #{ldap_servers_omnibus_config};
|
18
18
|
gitlab_rails['ldap_sync_worker_cron'] = '* * * * *';
|
@@ -15,7 +15,7 @@ module Gitlab
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def configure_omnibus(gitlab)
|
18
|
-
gitlab.
|
18
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
19
19
|
gitlab_rails['ldap_enabled'] = true;
|
20
20
|
gitlab_rails['ldap_servers'] = #{ldap_servers_omnibus_config};
|
21
21
|
gitlab_rails['ldap_sync_worker_cron'] = '* * * * *';
|
@@ -15,7 +15,7 @@ module Gitlab
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def configure_omnibus(gitlab)
|
18
|
-
gitlab.
|
18
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
19
19
|
gitlab_rails['ldap_enabled'] = true;
|
20
20
|
gitlab_rails['ldap_servers'] = #{ldap_servers_omnibus_config};
|
21
21
|
letsencrypt['enable'] = false;
|
@@ -13,7 +13,7 @@ module Gitlab
|
|
13
13
|
mattermost_external_url = "http://#{mattermost_hostname}"
|
14
14
|
|
15
15
|
gitlab.add_network_alias(mattermost_hostname)
|
16
|
-
gitlab.
|
16
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
17
17
|
mattermost_external_url '#{mattermost_external_url}';
|
18
18
|
OMNIBUS
|
19
19
|
|
@@ -21,7 +21,7 @@ module Gitlab
|
|
21
21
|
gitaly.network = @network
|
22
22
|
gitaly.skip_availability_check = true
|
23
23
|
|
24
|
-
gitaly.
|
24
|
+
gitaly.omnibus_configuration << gitaly_omnibus
|
25
25
|
gitaly.gitaly_tls
|
26
26
|
|
27
27
|
gitaly.instance do
|
@@ -30,7 +30,7 @@ module Gitlab
|
|
30
30
|
gitlab.name = @gitlab_name
|
31
31
|
gitlab.network = @network
|
32
32
|
|
33
|
-
gitlab.
|
33
|
+
gitlab.omnibus_configuration << gitlab_omnibus
|
34
34
|
gitlab.tls = true
|
35
35
|
|
36
36
|
gitlab.instance do
|
@@ -14,7 +14,7 @@ module Gitlab
|
|
14
14
|
attr_reader :gitlab_name, :spec_suite
|
15
15
|
|
16
16
|
def configure_omnibus(gitlab, mail_hog)
|
17
|
-
gitlab.
|
17
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
18
18
|
gitlab_rails['smtp_enable'] = true;
|
19
19
|
gitlab_rails['smtp_address'] = '#{mail_hog.hostname}';
|
20
20
|
gitlab_rails['smtp_port'] = 1025;
|
@@ -22,7 +22,7 @@ module Gitlab
|
|
22
22
|
tunnel_registry.gitlab_hostname = gitlab.hostname
|
23
23
|
tunnel_registry.network = 'test'
|
24
24
|
|
25
|
-
gitlab.
|
25
|
+
gitlab.omnibus_configuration << <<~OMNIBUS
|
26
26
|
external_url '#{tunnel_gitlab.url}';
|
27
27
|
nginx['listen_port'] = 80;
|
28
28
|
nginx['listen_https'] = false;
|
data/lib/gitlab/qa/version.rb
CHANGED