foreman_maintain 1.0.4 → 1.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/definitions/checks/maintenance_mode/check_consistency.rb +10 -2
- data/definitions/features/foreman_tasks.rb +14 -7
- data/definitions/features/instance.rb +4 -0
- data/definitions/features/iptables.rb +4 -21
- data/definitions/features/nftables.rb +51 -0
- data/definitions/procedures/content/fix_pulpcore_artifact_permissions.rb +30 -0
- data/definitions/procedures/maintenance_mode/disable_maintenance_mode.rb +18 -0
- data/definitions/procedures/maintenance_mode/enable_maintenance_mode.rb +48 -0
- data/definitions/procedures/maintenance_mode/is_enabled.rb +4 -2
- data/definitions/procedures/pulp/remove.rb +1 -0
- data/definitions/scenarios/content.rb +19 -0
- data/definitions/scenarios/self_upgrade.rb +12 -3
- data/definitions/scenarios/{upgrade_to_capsule_7_0.rb → upgrade_to_capsule_6_11.rb} +11 -10
- data/definitions/scenarios/{upgrade_to_capsule_7_0_z.rb → upgrade_to_capsule_6_11_z.rb} +11 -11
- data/definitions/scenarios/{upgrade_to_satellite_7_0.rb → upgrade_to_satellite_6_11.rb} +10 -10
- data/definitions/scenarios/{upgrade_to_satellite_7_0_z.rb → upgrade_to_satellite_6_11_z.rb} +11 -11
- data/lib/foreman_maintain/cli/content_command.rb +10 -0
- data/lib/foreman_maintain/cli/self_upgrade_command.rb +9 -24
- data/lib/foreman_maintain/concerns/downstream.rb +1 -1
- data/lib/foreman_maintain/concerns/firewall/iptables_maintenance_mode.rb +28 -0
- data/lib/foreman_maintain/concerns/firewall/nftables_maintenance_mode.rb +39 -0
- data/lib/foreman_maintain/version.rb +1 -1
- data/lib/foreman_maintain.rb +2 -0
- metadata +12 -8
- data/definitions/procedures/iptables/add_maintenance_mode_chain.rb +0 -15
- data/definitions/procedures/iptables/remove_maintenance_mode_chain.rb +0 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1db04c176ed0a5d5293dd9980db6c05e7f2c29c43f7da63d622e917c692d49ce
|
4
|
+
data.tar.gz: 2d41a5d195c217f7c7764b2543ed43ec3bacfac1339aad2162431052286775ca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d321a63d2e313db2694dbd70c138c498c829843b5ac891d33e91969a682494f94971b1411dbe81228f1e049bfb35b3a3ba2dc9c59d08629aa4297baff20210ad
|
7
|
+
data.tar.gz: a53ce7c727dbbf92d7257e7024587ba33a1e7db9b27c17fccf5e3ecc9ede63173bdcbcef2cd7f51173a3d88df661b6dd2e0f68fab05ea0983b8f657e9f02beae
|
@@ -22,11 +22,15 @@ module Checks::MaintenanceMode
|
|
22
22
|
|
23
23
|
private
|
24
24
|
|
25
|
+
def firewall
|
26
|
+
@firewall ||= feature(:instance).firewall
|
27
|
+
end
|
28
|
+
|
25
29
|
def verify_with_features
|
26
30
|
procedure_arr = []
|
27
31
|
feature_status_msgs = []
|
28
|
-
is_mode_on =
|
29
|
-
[
|
32
|
+
is_mode_on = firewall.maintenance_mode_status?
|
33
|
+
[firewall.label, :sync_plans, :cron].each do |feature_name|
|
30
34
|
msg, procedures_to_run = send("check_for_#{feature_name}", is_mode_on)
|
31
35
|
feature_status_msgs << msg
|
32
36
|
procedure_arr.concat(procedures_to_run)
|
@@ -55,6 +59,10 @@ module Checks::MaintenanceMode
|
|
55
59
|
feature(:iptables).status_for_maintenance_mode
|
56
60
|
end
|
57
61
|
|
62
|
+
def check_for_nftables(_is_mode_on)
|
63
|
+
feature(:nftables).status_for_maintenance_mode
|
64
|
+
end
|
65
|
+
|
58
66
|
def check_for_sync_plans(is_mode_on)
|
59
67
|
feature(:sync_plans).status_for_maintenance_mode(is_mode_on)
|
60
68
|
end
|
@@ -82,15 +82,22 @@ class Features::ForemanTasks < ForemanMaintain::Feature
|
|
82
82
|
def delete(state)
|
83
83
|
tasks_condition = condition(state)
|
84
84
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
85
|
+
sql = <<-SQL
|
86
|
+
DELETE FROM dynflow_steps USING foreman_tasks_tasks WHERE (foreman_tasks_tasks.external_id = dynflow_steps.execution_plan_uuid::varchar) AND #{tasks_condition};
|
87
|
+
DELETE FROM dynflow_actions USING foreman_tasks_tasks WHERE (foreman_tasks_tasks.external_id = dynflow_actions.execution_plan_uuid::varchar) AND #{tasks_condition};
|
88
|
+
DELETE FROM dynflow_execution_plans USING foreman_tasks_tasks WHERE (foreman_tasks_tasks.external_id = dynflow_execution_plans.uuid::varchar) AND #{tasks_condition};
|
89
|
+
DELETE FROM foreman_tasks_tasks WHERE #{tasks_condition};
|
90
|
+
-- Delete locks and links which may now be orphaned
|
91
|
+
DELETE FROM foreman_tasks_locks as ftl where ftl.task_id NOT IN (SELECT id FROM foreman_tasks_tasks);
|
92
92
|
SQL
|
93
93
|
|
94
|
+
if check_min_version(foreman_plugin_name('foreman-tasks'), '4.0.0')
|
95
|
+
sql += 'DELETE FROM foreman_tasks_links as ftl ' \
|
96
|
+
'where ftl.task_id NOT IN (SELECT id FROM foreman_tasks_tasks);'
|
97
|
+
end
|
98
|
+
|
99
|
+
feature(:foreman_database).psql("BEGIN; #{sql}; COMMIT;")
|
100
|
+
|
94
101
|
count(state)
|
95
102
|
end
|
96
103
|
|
@@ -1,6 +1,10 @@
|
|
1
1
|
class Features::Iptables < ForemanMaintain::Feature
|
2
|
+
include ForemanMaintain::Concerns::Firewall::IptablesMaintenanceMode
|
2
3
|
metadata do
|
3
4
|
label :iptables
|
5
|
+
confine do
|
6
|
+
find_package('iptables')
|
7
|
+
end
|
4
8
|
end
|
5
9
|
|
6
10
|
def add_chain(chain_name, rules, rule_chain = 'INPUT')
|
@@ -29,27 +33,6 @@ class Features::Iptables < ForemanMaintain::Feature
|
|
29
33
|
execute?("iptables -L #{rule_chain} | tail -n +3 | grep '^#{target_name} '")
|
30
34
|
end
|
31
35
|
|
32
|
-
def add_maintenance_mode_chain
|
33
|
-
add_chain(custom_chain_name,
|
34
|
-
['-i lo -j ACCEPT', '-p tcp --dport 443 -j REJECT'])
|
35
|
-
end
|
36
|
-
|
37
|
-
def remove_maintenance_mode_chain
|
38
|
-
remove_chain(custom_chain_name)
|
39
|
-
end
|
40
|
-
|
41
|
-
def maintenance_mode_chain_exist?
|
42
|
-
chain_exist?(custom_chain_name)
|
43
|
-
end
|
44
|
-
|
45
|
-
def status_for_maintenance_mode
|
46
|
-
if maintenance_mode_chain_exist?
|
47
|
-
['Iptables chain: present', []]
|
48
|
-
else
|
49
|
-
['Iptables chain: absent', []]
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
36
|
private
|
54
37
|
|
55
38
|
def custom_chain_name
|
@@ -0,0 +1,51 @@
|
|
1
|
+
class Features::Nftables < ForemanMaintain::Feature
|
2
|
+
include ForemanMaintain::Concerns::Firewall::NftablesMaintenanceMode
|
3
|
+
metadata do
|
4
|
+
label :nftables
|
5
|
+
confine do
|
6
|
+
find_package('nftables')
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
def add_table(options = '')
|
11
|
+
options = "#{ip_family} #{table_name}" if options.empty?
|
12
|
+
execute!("nft add table #{options}")
|
13
|
+
end
|
14
|
+
|
15
|
+
def delete_table(options = '')
|
16
|
+
options = "#{ip_family} #{table_name}" if options.empty?
|
17
|
+
execute!("nft delete table #{options}")
|
18
|
+
end
|
19
|
+
|
20
|
+
def add_chain(options = {})
|
21
|
+
family = options.fetch(:family, ip_family)
|
22
|
+
table = options.fetch(:table, table_name)
|
23
|
+
chain = options.fetch(:chain, chain_name)
|
24
|
+
chain_options = options.fetch(:chain_options)
|
25
|
+
execute!("nft add chain #{family} #{table} #{chain} #{chain_options}")
|
26
|
+
end
|
27
|
+
|
28
|
+
def add_rule(options = {})
|
29
|
+
family = options.fetch(:family, ip_family)
|
30
|
+
table = options.fetch(:table, table_name)
|
31
|
+
chain = options.fetch(:chain, chain_name)
|
32
|
+
rule = options.fetch(:rule) # needs validation
|
33
|
+
execute!("nft add rule #{family} #{table} #{chain} #{rule}")
|
34
|
+
end
|
35
|
+
|
36
|
+
def table_exist?(name = table_name)
|
37
|
+
execute!('nft list tables').include?(name)
|
38
|
+
end
|
39
|
+
|
40
|
+
def table_name
|
41
|
+
'FOREMAN_MAINTAIN_TABLE'
|
42
|
+
end
|
43
|
+
|
44
|
+
def chain_name
|
45
|
+
'FOREMAN_MAINTAIN_CHAIN'
|
46
|
+
end
|
47
|
+
|
48
|
+
def ip_family
|
49
|
+
'inet'
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Procedures::Content
|
2
|
+
class FixPulpcoreArtifactOwnership < ForemanMaintain::Procedure
|
3
|
+
metadata do
|
4
|
+
description 'Fix Pulpcore artifact ownership to be pulp:pulp'
|
5
|
+
param :assumeyes, 'Do not ask for confirmation', :default => false
|
6
|
+
|
7
|
+
confine do
|
8
|
+
check_min_version(foreman_plugin_name('katello'), '4.0')
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def ask_to_proceed
|
13
|
+
question = "\nWARNING: Only proceed if your system is fully switched to Pulp 3.\n"
|
14
|
+
question += "\n\nDo you want to proceed?"
|
15
|
+
answer = ask_decision(question, actions_msg: 'y(yes), q(quit)')
|
16
|
+
abort! if answer != :yes
|
17
|
+
end
|
18
|
+
|
19
|
+
def run
|
20
|
+
assumeyes_val = @assumeyes.nil? ? assumeyes? : @assumeyes
|
21
|
+
|
22
|
+
ask_to_proceed unless assumeyes_val
|
23
|
+
|
24
|
+
with_spinner('Updating artifact ownership for Pulp 3') do |spinner|
|
25
|
+
spinner.update('# chown -hR pulp.pulp /var/lib/pulp/media/artifact')
|
26
|
+
FileUtils.chown_R 'pulp', 'pulp', '/var/lib/pulp/media/artifact'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Procedures::MaintenanceMode
|
2
|
+
class DisableMaintenanceMode < ForemanMaintain::Procedure
|
3
|
+
metadata do
|
4
|
+
label :disable_maintenance_mode
|
5
|
+
description 'Remove maintenance mode table/chain from nftables/iptables'
|
6
|
+
tags :post_migrations, :maintenance_mode_off
|
7
|
+
after :sync_plans_enable
|
8
|
+
end
|
9
|
+
|
10
|
+
def run
|
11
|
+
if feature(:instance).firewall
|
12
|
+
feature(:instance).firewall.disable_maintenance_mode
|
13
|
+
else
|
14
|
+
warn! 'Unable to find nftables or iptables'
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module Procedures::MaintenanceMode
|
2
|
+
class EnableMaintenanceMode < ForemanMaintain::Procedure
|
3
|
+
metadata do
|
4
|
+
label :enable_maintenance_mode
|
5
|
+
description 'Add maintenance_mode tables/chain to nftables/iptables'
|
6
|
+
tags :pre_migrations, :maintenance_mode_on
|
7
|
+
after :sync_plans_disable
|
8
|
+
end
|
9
|
+
|
10
|
+
def run
|
11
|
+
if feature(:instance).firewall
|
12
|
+
feature(:instance).firewall.enable_maintenance_mode
|
13
|
+
else
|
14
|
+
notify_and_ask_to_install_firewall_utility
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def notify_and_ask_to_install_firewall_utility
|
19
|
+
puts 'Unable to find nftables or iptables!'
|
20
|
+
question, pkg = question_and_pkg_name
|
21
|
+
answer = ask_decision(question, actions_msg: 'y(yes), q(quit)')
|
22
|
+
if answer == :yes
|
23
|
+
packages_action(:install, pkg)
|
24
|
+
feature(:instance).firewall.enable_maintenance_mode
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def can_install_nft?
|
29
|
+
nft_kernel_version = Gem::Version.new('3.13')
|
30
|
+
installed_kernel_version = Gem::Version.new(execute!('uname -r').split('-').first)
|
31
|
+
installed_kernel_version >= nft_kernel_version
|
32
|
+
end
|
33
|
+
|
34
|
+
def question_and_pkg_name
|
35
|
+
question = 'Do you want to install missing netfilter utility '
|
36
|
+
pkg_to_install = []
|
37
|
+
if can_install_nft?
|
38
|
+
question << 'nftables?'
|
39
|
+
pkg_to_install << 'nftables'
|
40
|
+
else
|
41
|
+
question << 'iptables?'
|
42
|
+
pkg_to_install << 'iptables'
|
43
|
+
end
|
44
|
+
question << "\nand start maintenance mode?"
|
45
|
+
[question, pkg_to_install]
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -2,14 +2,16 @@ module Procedures::MaintenanceMode
|
|
2
2
|
class IsEnabled < ForemanMaintain::Procedure
|
3
3
|
metadata do
|
4
4
|
description 'Showing status code for maintenance_mode'
|
5
|
-
for_feature :iptables
|
6
5
|
advanced_run false
|
6
|
+
confine do
|
7
|
+
feature(:nftables) || feature(:iptables)
|
8
|
+
end
|
7
9
|
end
|
8
10
|
|
9
11
|
attr_reader :status_code
|
10
12
|
|
11
13
|
def run
|
12
|
-
@status_code = feature(:
|
14
|
+
@status_code = feature(:instance).firewall.maintenance_mode_status? ? 0 : 1
|
13
15
|
puts "Maintenance mode is #{@status_code == 1 ? 'Off' : 'On'}"
|
14
16
|
end
|
15
17
|
end
|
@@ -129,10 +129,29 @@ module ForemanMaintain::Scenarios
|
|
129
129
|
|
130
130
|
def set_context_mapping
|
131
131
|
context.map(:assumeyes, Procedures::Pulp::Remove => :assumeyes)
|
132
|
+
context.map(:assumeyes, Procedures::Content::FixPulpcoreArtifactOwnership => :assumeyes)
|
132
133
|
end
|
133
134
|
|
134
135
|
def compose
|
135
136
|
add_step_with_context(Procedures::Pulp::Remove)
|
137
|
+
add_step_with_context(Procedures::Content::FixPulpcoreArtifactOwnership)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
class FixPulpcoreArtifactOwnership < ContentBase
|
142
|
+
metadata do
|
143
|
+
label :content_fix_pulpcore_artifact_ownership
|
144
|
+
description 'Fix Pulpcore artifact ownership to be pulp:pulp'
|
145
|
+
param :assumeyes, 'Do not ask for confirmation'
|
146
|
+
manual_detection
|
147
|
+
end
|
148
|
+
|
149
|
+
def set_context_mapping
|
150
|
+
context.map(:assumeyes, Procedures::Content::FixPulpcoreArtifactOwnership => :assumeyes)
|
151
|
+
end
|
152
|
+
|
153
|
+
def compose
|
154
|
+
add_step_with_context(Procedures::Content::FixPulpcoreArtifactOwnership)
|
136
155
|
end
|
137
156
|
end
|
138
157
|
end
|
@@ -13,15 +13,22 @@ module ForemanMaintain::Scenarios
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def target_version
|
16
|
-
|
16
|
+
current_full_version = feature(:instance).downstream.current_version
|
17
|
+
@target_version ||= current_full_version.bump
|
17
18
|
end
|
18
19
|
|
19
20
|
def current_version
|
20
21
|
feature(:instance).downstream.current_minor_version
|
21
22
|
end
|
22
23
|
|
24
|
+
def maintenance_repo_label
|
25
|
+
@maintenance_repo_label ||= context.get(:maintenance_repo_label)
|
26
|
+
end
|
27
|
+
|
23
28
|
def maintenance_repo_id(version)
|
24
|
-
if
|
29
|
+
if maintenance_repo_label
|
30
|
+
return maintenance_repo_label
|
31
|
+
elsif (repo = ENV['MAINTENANCE_REPO_LABEL'])
|
25
32
|
return repo unless repo.empty?
|
26
33
|
end
|
27
34
|
|
@@ -64,7 +71,9 @@ module ForemanMaintain::Scenarios
|
|
64
71
|
end
|
65
72
|
|
66
73
|
def use_rhsm?
|
67
|
-
|
74
|
+
return false if maintenance_repo_label
|
75
|
+
|
76
|
+
if (repo = ENV['MAINTENANCE_REPO_LABEL'])
|
68
77
|
return false unless repo.empty?
|
69
78
|
end
|
70
79
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
module Scenarios::
|
1
|
+
module Scenarios::Capsule_6_11
|
2
2
|
class Abstract < ForemanMaintain::Scenario
|
3
3
|
def self.upgrade_metadata(&block)
|
4
4
|
metadata do
|
@@ -6,20 +6,20 @@ module Scenarios::Capsule_7_0
|
|
6
6
|
confine do
|
7
7
|
feature(:capsule) &&
|
8
8
|
(feature(:capsule).current_minor_version == '6.10' || \
|
9
|
-
ForemanMaintain.upgrade_in_progress == '
|
9
|
+
ForemanMaintain.upgrade_in_progress == '6.11')
|
10
10
|
end
|
11
11
|
instance_eval(&block)
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
15
|
def target_version
|
16
|
-
'
|
16
|
+
'6.11'
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
20
|
class PreUpgradeCheck < Abstract
|
21
21
|
upgrade_metadata do
|
22
|
-
description 'Checks before upgrading to Capsule
|
22
|
+
description 'Checks before upgrading to Capsule 6.11'
|
23
23
|
tags :pre_upgrade_checks
|
24
24
|
run_strategy :fail_slow
|
25
25
|
end
|
@@ -27,25 +27,26 @@ module Scenarios::Capsule_7_0
|
|
27
27
|
def compose
|
28
28
|
add_steps(find_checks(:default))
|
29
29
|
add_steps(find_checks(:pre_upgrade))
|
30
|
-
add_step(Checks::Repositories::Validate.new(:version => '
|
30
|
+
add_step(Checks::Repositories::Validate.new(:version => '6.11'))
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
34
34
|
class PreMigrations < Abstract
|
35
35
|
upgrade_metadata do
|
36
|
-
description 'Procedures before migrating to Capsule
|
36
|
+
description 'Procedures before migrating to Capsule 6.11'
|
37
37
|
tags :pre_migrations
|
38
38
|
end
|
39
39
|
|
40
40
|
def compose
|
41
41
|
add_steps(find_procedures(:pre_migrations))
|
42
|
+
add_step(Procedures::Pulp::Remove.new(:assumeyes => true))
|
42
43
|
add_step(Procedures::Service::Stop.new)
|
43
44
|
end
|
44
45
|
end
|
45
46
|
|
46
47
|
class Migrations < Abstract
|
47
48
|
upgrade_metadata do
|
48
|
-
description 'Migration scripts to Capsule
|
49
|
+
description 'Migration scripts to Capsule 6.11'
|
49
50
|
tags :migrations
|
50
51
|
end
|
51
52
|
|
@@ -54,7 +55,7 @@ module Scenarios::Capsule_7_0
|
|
54
55
|
end
|
55
56
|
|
56
57
|
def compose
|
57
|
-
add_step(Procedures::Repositories::Setup.new(:version => '
|
58
|
+
add_step(Procedures::Repositories::Setup.new(:version => '6.11'))
|
58
59
|
add_step(Procedures::Packages::UnlockVersions.new)
|
59
60
|
add_step(Procedures::Packages::Update.new(:assumeyes => true))
|
60
61
|
add_step_with_context(Procedures::Installer::Upgrade)
|
@@ -63,7 +64,7 @@ module Scenarios::Capsule_7_0
|
|
63
64
|
|
64
65
|
class PostMigrations < Abstract
|
65
66
|
upgrade_metadata do
|
66
|
-
description 'Procedures after migrating to Capsule
|
67
|
+
description 'Procedures after migrating to Capsule 6.11'
|
67
68
|
tags :post_migrations
|
68
69
|
end
|
69
70
|
|
@@ -76,7 +77,7 @@ module Scenarios::Capsule_7_0
|
|
76
77
|
|
77
78
|
class PostUpgradeChecks < Abstract
|
78
79
|
upgrade_metadata do
|
79
|
-
description 'Checks after upgrading to Capsule
|
80
|
+
description 'Checks after upgrading to Capsule 6.11'
|
80
81
|
tags :post_upgrade_checks
|
81
82
|
run_strategy :fail_slow
|
82
83
|
end
|
@@ -1,25 +1,25 @@
|
|
1
|
-
module Scenarios::
|
1
|
+
module Scenarios::Capsule_6_11_z
|
2
2
|
class Abstract < ForemanMaintain::Scenario
|
3
3
|
def self.upgrade_metadata(&block)
|
4
4
|
metadata do
|
5
5
|
tags :upgrade_scenario
|
6
6
|
confine do
|
7
7
|
feature(:capsule) &&
|
8
|
-
(feature(:capsule).current_minor_version == '
|
9
|
-
ForemanMaintain.upgrade_in_progress == '
|
8
|
+
(feature(:capsule).current_minor_version == '6.11' || \
|
9
|
+
ForemanMaintain.upgrade_in_progress == '6.11.z')
|
10
10
|
end
|
11
11
|
instance_eval(&block)
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
15
|
def target_version
|
16
|
-
'
|
16
|
+
'6.11.z'
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
20
|
class PreUpgradeCheck < Abstract
|
21
21
|
upgrade_metadata do
|
22
|
-
description 'Checks before upgrading to Capsule
|
22
|
+
description 'Checks before upgrading to Capsule 6.11.z'
|
23
23
|
tags :pre_upgrade_checks
|
24
24
|
run_strategy :fail_slow
|
25
25
|
end
|
@@ -27,13 +27,13 @@ module Scenarios::Capsule_7_0_z
|
|
27
27
|
def compose
|
28
28
|
add_steps(find_checks(:default))
|
29
29
|
add_steps(find_checks(:pre_upgrade))
|
30
|
-
add_step(Checks::Repositories::Validate.new(:version => '
|
30
|
+
add_step(Checks::Repositories::Validate.new(:version => '6.11'))
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
34
34
|
class PreMigrations < Abstract
|
35
35
|
upgrade_metadata do
|
36
|
-
description 'Procedures before migrating to Capsule
|
36
|
+
description 'Procedures before migrating to Capsule 6.11.z'
|
37
37
|
tags :pre_migrations
|
38
38
|
end
|
39
39
|
|
@@ -45,7 +45,7 @@ module Scenarios::Capsule_7_0_z
|
|
45
45
|
|
46
46
|
class Migrations < Abstract
|
47
47
|
upgrade_metadata do
|
48
|
-
description 'Migration scripts to Capsule
|
48
|
+
description 'Migration scripts to Capsule 6.11.z'
|
49
49
|
tags :migrations
|
50
50
|
end
|
51
51
|
|
@@ -54,7 +54,7 @@ module Scenarios::Capsule_7_0_z
|
|
54
54
|
end
|
55
55
|
|
56
56
|
def compose
|
57
|
-
add_step(Procedures::Repositories::Setup.new(:version => '
|
57
|
+
add_step(Procedures::Repositories::Setup.new(:version => '6.11'))
|
58
58
|
add_step(Procedures::Packages::UnlockVersions.new)
|
59
59
|
add_step(Procedures::Packages::Update.new(:assumeyes => true))
|
60
60
|
add_step_with_context(Procedures::Installer::Upgrade)
|
@@ -63,7 +63,7 @@ module Scenarios::Capsule_7_0_z
|
|
63
63
|
|
64
64
|
class PostMigrations < Abstract
|
65
65
|
upgrade_metadata do
|
66
|
-
description 'Procedures after migrating to Capsule
|
66
|
+
description 'Procedures after migrating to Capsule 6.11.z'
|
67
67
|
tags :post_migrations
|
68
68
|
end
|
69
69
|
|
@@ -76,7 +76,7 @@ module Scenarios::Capsule_7_0_z
|
|
76
76
|
|
77
77
|
class PostUpgradeChecks < Abstract
|
78
78
|
upgrade_metadata do
|
79
|
-
description 'Checks after upgrading to Capsule
|
79
|
+
description 'Checks after upgrading to Capsule 6.11.z'
|
80
80
|
tags :post_upgrade_checks
|
81
81
|
run_strategy :fail_slow
|
82
82
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
module Scenarios::
|
1
|
+
module Scenarios::Satellite_6_11
|
2
2
|
class Abstract < ForemanMaintain::Scenario
|
3
3
|
def self.upgrade_metadata(&block)
|
4
4
|
metadata do
|
@@ -6,20 +6,20 @@ module Scenarios::Satellite_7_0
|
|
6
6
|
confine do
|
7
7
|
feature(:satellite) &&
|
8
8
|
(feature(:satellite).current_minor_version == '6.10' || \
|
9
|
-
ForemanMaintain.upgrade_in_progress == '
|
9
|
+
ForemanMaintain.upgrade_in_progress == '6.11')
|
10
10
|
end
|
11
11
|
instance_eval(&block)
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
15
|
def target_version
|
16
|
-
'
|
16
|
+
'6.11'
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
20
|
class PreUpgradeCheck < Abstract
|
21
21
|
upgrade_metadata do
|
22
|
-
description 'Checks before upgrading to Satellite
|
22
|
+
description 'Checks before upgrading to Satellite 6.11'
|
23
23
|
tags :pre_upgrade_checks
|
24
24
|
run_strategy :fail_slow
|
25
25
|
end
|
@@ -29,13 +29,13 @@ module Scenarios::Satellite_7_0
|
|
29
29
|
add_steps(find_checks(:pre_upgrade))
|
30
30
|
|
31
31
|
add_step(Checks::Foreman::CheckpointSegments)
|
32
|
-
add_step(Checks::Repositories::Validate.new(:version => '
|
32
|
+
add_step(Checks::Repositories::Validate.new(:version => '6.11'))
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
36
|
class PreMigrations < Abstract
|
37
37
|
upgrade_metadata do
|
38
|
-
description 'Procedures before migrating to Satellite
|
38
|
+
description 'Procedures before migrating to Satellite 6.11'
|
39
39
|
tags :pre_migrations
|
40
40
|
end
|
41
41
|
|
@@ -48,7 +48,7 @@ module Scenarios::Satellite_7_0
|
|
48
48
|
|
49
49
|
class Migrations < Abstract
|
50
50
|
upgrade_metadata do
|
51
|
-
description 'Migration scripts to Satellite
|
51
|
+
description 'Migration scripts to Satellite 6.11'
|
52
52
|
tags :migrations
|
53
53
|
run_strategy :fail_fast
|
54
54
|
end
|
@@ -58,7 +58,7 @@ module Scenarios::Satellite_7_0
|
|
58
58
|
end
|
59
59
|
|
60
60
|
def compose
|
61
|
-
add_step(Procedures::Repositories::Setup.new(:version => '
|
61
|
+
add_step(Procedures::Repositories::Setup.new(:version => '6.11'))
|
62
62
|
add_step(Procedures::Packages::UnlockVersions.new)
|
63
63
|
add_step(Procedures::Packages::Update.new(:assumeyes => true))
|
64
64
|
add_step_with_context(Procedures::Installer::Upgrade)
|
@@ -68,7 +68,7 @@ module Scenarios::Satellite_7_0
|
|
68
68
|
|
69
69
|
class PostMigrations < Abstract
|
70
70
|
upgrade_metadata do
|
71
|
-
description 'Procedures after migrating to Satellite
|
71
|
+
description 'Procedures after migrating to Satellite 6.11'
|
72
72
|
tags :post_migrations
|
73
73
|
end
|
74
74
|
|
@@ -81,7 +81,7 @@ module Scenarios::Satellite_7_0
|
|
81
81
|
|
82
82
|
class PostUpgradeChecks < Abstract
|
83
83
|
upgrade_metadata do
|
84
|
-
description 'Checks after upgrading to Satellite
|
84
|
+
description 'Checks after upgrading to Satellite 6.11'
|
85
85
|
tags :post_upgrade_checks
|
86
86
|
run_strategy :fail_slow
|
87
87
|
end
|
@@ -1,25 +1,25 @@
|
|
1
|
-
module Scenarios::
|
1
|
+
module Scenarios::Satellite_6_11_z
|
2
2
|
class Abstract < ForemanMaintain::Scenario
|
3
3
|
def self.upgrade_metadata(&block)
|
4
4
|
metadata do
|
5
5
|
tags :upgrade_scenario
|
6
6
|
confine do
|
7
7
|
feature(:satellite) &&
|
8
|
-
(feature(:satellite).current_minor_version == '
|
9
|
-
ForemanMaintain.upgrade_in_progress == '
|
8
|
+
(feature(:satellite).current_minor_version == '6.11' || \
|
9
|
+
ForemanMaintain.upgrade_in_progress == '6.11.z')
|
10
10
|
end
|
11
11
|
instance_eval(&block)
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
15
|
def target_version
|
16
|
-
'
|
16
|
+
'6.11.z'
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
20
|
class PreUpgradeCheck < Abstract
|
21
21
|
upgrade_metadata do
|
22
|
-
description 'Checks before upgrading to Satellite
|
22
|
+
description 'Checks before upgrading to Satellite 6.11.z'
|
23
23
|
tags :pre_upgrade_checks
|
24
24
|
run_strategy :fail_slow
|
25
25
|
end
|
@@ -27,13 +27,13 @@ module Scenarios::Satellite_7_0_z
|
|
27
27
|
def compose
|
28
28
|
add_steps(find_checks(:default))
|
29
29
|
add_steps(find_checks(:pre_upgrade))
|
30
|
-
add_step(Checks::Repositories::Validate.new(:version => '
|
30
|
+
add_step(Checks::Repositories::Validate.new(:version => '6.11'))
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
34
34
|
class PreMigrations < Abstract
|
35
35
|
upgrade_metadata do
|
36
|
-
description 'Procedures before migrating to Satellite
|
36
|
+
description 'Procedures before migrating to Satellite 6.11.z'
|
37
37
|
tags :pre_migrations
|
38
38
|
end
|
39
39
|
|
@@ -45,7 +45,7 @@ module Scenarios::Satellite_7_0_z
|
|
45
45
|
|
46
46
|
class Migrations < Abstract
|
47
47
|
upgrade_metadata do
|
48
|
-
description 'Migration scripts to Satellite
|
48
|
+
description 'Migration scripts to Satellite 6.11.z'
|
49
49
|
tags :migrations
|
50
50
|
end
|
51
51
|
|
@@ -54,7 +54,7 @@ module Scenarios::Satellite_7_0_z
|
|
54
54
|
end
|
55
55
|
|
56
56
|
def compose
|
57
|
-
add_step(Procedures::Repositories::Setup.new(:version => '
|
57
|
+
add_step(Procedures::Repositories::Setup.new(:version => '6.11'))
|
58
58
|
add_step(Procedures::Packages::UnlockVersions.new)
|
59
59
|
add_step(Procedures::Packages::Update.new(:assumeyes => true))
|
60
60
|
add_step_with_context(Procedures::Installer::Upgrade)
|
@@ -64,7 +64,7 @@ module Scenarios::Satellite_7_0_z
|
|
64
64
|
|
65
65
|
class PostMigrations < Abstract
|
66
66
|
upgrade_metadata do
|
67
|
-
description 'Procedures after migrating to Satellite
|
67
|
+
description 'Procedures after migrating to Satellite 6.11.z'
|
68
68
|
tags :post_migrations
|
69
69
|
end
|
70
70
|
|
@@ -77,7 +77,7 @@ module Scenarios::Satellite_7_0_z
|
|
77
77
|
|
78
78
|
class PostUpgradeChecks < Abstract
|
79
79
|
upgrade_metadata do
|
80
|
-
description 'Checks after upgrading to Satellite
|
80
|
+
description 'Checks after upgrading to Satellite 6.11.z'
|
81
81
|
tags :post_upgrade_checks
|
82
82
|
run_strategy :fail_slow
|
83
83
|
end
|
@@ -54,6 +54,16 @@ module ForemanMaintain
|
|
54
54
|
)
|
55
55
|
end
|
56
56
|
end
|
57
|
+
|
58
|
+
subcommand 'fix-pulpcore-artifact-ownership',
|
59
|
+
'Update filesystem ownership for Pulpcore artifacts' do
|
60
|
+
interactive_option(%w[assumeyes plaintext])
|
61
|
+
def execute
|
62
|
+
run_scenarios_and_exit(
|
63
|
+
Scenarios::Content::FixPulpcoreArtifactOwnership.new(:assumeyes => assumeyes?)
|
64
|
+
)
|
65
|
+
end
|
66
|
+
end
|
57
67
|
end
|
58
68
|
end
|
59
69
|
end
|
@@ -1,38 +1,23 @@
|
|
1
1
|
module ForemanMaintain
|
2
2
|
module Cli
|
3
3
|
class SelfUpgradeCommand < Base
|
4
|
-
option ['--
|
5
|
-
'
|
6
|
-
|
4
|
+
option ['--maintenance-repo-label'], 'REPOSITORY_LABEL',\
|
5
|
+
'Repository label from which packages should be updated.'\
|
6
|
+
'This can be used when standard CDN repositories are unavailable.'
|
7
7
|
def execute
|
8
|
-
allow_major_version_upgrade_only
|
9
8
|
run_scenario(upgrade_scenario, upgrade_rescue_scenario)
|
10
9
|
end
|
11
10
|
|
12
11
|
def upgrade_scenario
|
13
|
-
Scenarios::SelfUpgrade.new(
|
12
|
+
Scenarios::SelfUpgrade.new(
|
13
|
+
maintenance_repo_label: maintenance_repo_label
|
14
|
+
)
|
14
15
|
end
|
15
16
|
|
16
17
|
def upgrade_rescue_scenario
|
17
|
-
Scenarios::SelfUpgradeRescue.new(
|
18
|
-
|
19
|
-
|
20
|
-
def current_downstream_version
|
21
|
-
ForemanMaintain.detector.feature(:instance).downstream.current_version
|
22
|
-
end
|
23
|
-
|
24
|
-
def allow_major_version_upgrade_only
|
25
|
-
begin
|
26
|
-
next_version = Gem::Version.new(target_version)
|
27
|
-
rescue ArgumentError => err
|
28
|
-
raise Error::UsageError, "Invalid version! #{err}"
|
29
|
-
end
|
30
|
-
if current_downstream_version >= next_version
|
31
|
-
message = "The target-version #{target_version} should be "\
|
32
|
-
"greater than existing version #{current_downstream_version},"\
|
33
|
-
"\nand self-upgrade should be used for major version upgrades only!"
|
34
|
-
raise Error::UsageError, message
|
35
|
-
end
|
18
|
+
Scenarios::SelfUpgradeRescue.new(
|
19
|
+
maintenance_repo_label: maintenance_repo_label
|
20
|
+
)
|
36
21
|
end
|
37
22
|
end
|
38
23
|
end
|
@@ -116,7 +116,7 @@ module ForemanMaintain
|
|
116
116
|
end
|
117
117
|
|
118
118
|
def common_repos(full_version)
|
119
|
-
sat_maint_version = if version(full_version) >= version('
|
119
|
+
sat_maint_version = if version(full_version) >= version('6.11') && !use_beta_repos?
|
120
120
|
full_version
|
121
121
|
else
|
122
122
|
full_version[0]
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module ForemanMaintain
|
2
|
+
module Concerns
|
3
|
+
module Firewall
|
4
|
+
module IptablesMaintenanceMode
|
5
|
+
def disable_maintenance_mode
|
6
|
+
remove_chain(custom_chain_name)
|
7
|
+
end
|
8
|
+
|
9
|
+
def enable_maintenance_mode
|
10
|
+
add_chain(custom_chain_name,
|
11
|
+
['-i lo -j ACCEPT', '-p tcp --dport 443 -j REJECT'])
|
12
|
+
end
|
13
|
+
|
14
|
+
def maintenance_mode_status?
|
15
|
+
chain_exist?(custom_chain_name)
|
16
|
+
end
|
17
|
+
|
18
|
+
def status_for_maintenance_mode
|
19
|
+
if maintenance_mode_status?
|
20
|
+
['Iptables chain: present', []]
|
21
|
+
else
|
22
|
+
['Iptables chain: absent', []]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module ForemanMaintain
|
2
|
+
module Concerns
|
3
|
+
module Firewall
|
4
|
+
module NftablesMaintenanceMode
|
5
|
+
def disable_maintenance_mode
|
6
|
+
delete_table if table_exist?
|
7
|
+
end
|
8
|
+
|
9
|
+
def enable_maintenance_mode
|
10
|
+
unless table_exist?
|
11
|
+
add_table
|
12
|
+
add_chain(:chain_options => nftables_chain_options)
|
13
|
+
add_rule(rule: nftables_rule)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def maintenance_mode_status?
|
18
|
+
table_exist?
|
19
|
+
end
|
20
|
+
|
21
|
+
def nftables_chain_options
|
22
|
+
'{type filter hook input priority 0\\;}'
|
23
|
+
end
|
24
|
+
|
25
|
+
def nftables_rule
|
26
|
+
'tcp dport https reject'
|
27
|
+
end
|
28
|
+
|
29
|
+
def status_for_maintenance_mode
|
30
|
+
if table_exist?
|
31
|
+
['Nftables table: present', []]
|
32
|
+
else
|
33
|
+
['Nftables table: absent', []]
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/foreman_maintain.rb
CHANGED
@@ -24,6 +24,8 @@ module ForemanMaintain
|
|
24
24
|
require 'foreman_maintain/concerns/downstream'
|
25
25
|
require 'foreman_maintain/concerns/primary_checks'
|
26
26
|
require 'foreman_maintain/concerns/pulp_common'
|
27
|
+
require 'foreman_maintain/concerns/firewall/iptables_maintenance_mode'
|
28
|
+
require 'foreman_maintain/concerns/firewall/nftables_maintenance_mode'
|
27
29
|
require 'foreman_maintain/top_level_modules'
|
28
30
|
require 'foreman_maintain/yaml_storage'
|
29
31
|
require 'foreman_maintain/config'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: foreman_maintain
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ivan Nečas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-04-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: clamp
|
@@ -206,6 +206,7 @@ files:
|
|
206
206
|
- definitions/features/iptables.rb
|
207
207
|
- definitions/features/katello.rb
|
208
208
|
- definitions/features/mongo.rb
|
209
|
+
- definitions/features/nftables.rb
|
209
210
|
- definitions/features/pulp2.rb
|
210
211
|
- definitions/features/pulpcore.rb
|
211
212
|
- definitions/features/pulpcore_database.rb
|
@@ -244,6 +245,7 @@ files:
|
|
244
245
|
- definitions/procedures/backup/snapshot/mount_pulpcore_db.rb
|
245
246
|
- definitions/procedures/backup/snapshot/prepare_mount.rb
|
246
247
|
- definitions/procedures/candlepin/delete_orphaned_records_from_env_content.rb
|
248
|
+
- definitions/procedures/content/fix_pulpcore_artifact_permissions.rb
|
247
249
|
- definitions/procedures/content/migration_reset.rb
|
248
250
|
- definitions/procedures/content/migration_stats.rb
|
249
251
|
- definitions/procedures/content/prepare.rb
|
@@ -267,9 +269,9 @@ files:
|
|
267
269
|
- definitions/procedures/installer/run.rb
|
268
270
|
- definitions/procedures/installer/upgrade.rb
|
269
271
|
- definitions/procedures/installer/upgrade_rake_task.rb
|
270
|
-
- definitions/procedures/iptables/add_maintenance_mode_chain.rb
|
271
|
-
- definitions/procedures/iptables/remove_maintenance_mode_chain.rb
|
272
272
|
- definitions/procedures/knowledge_base_article.rb
|
273
|
+
- definitions/procedures/maintenance_mode/disable_maintenance_mode.rb
|
274
|
+
- definitions/procedures/maintenance_mode/enable_maintenance_mode.rb
|
273
275
|
- definitions/procedures/maintenance_mode/is_enabled.rb
|
274
276
|
- definitions/procedures/packages/check_update.rb
|
275
277
|
- definitions/procedures/packages/enable_version_locking.rb
|
@@ -332,14 +334,16 @@ files:
|
|
332
334
|
- definitions/scenarios/services.rb
|
333
335
|
- definitions/scenarios/upgrade_to_capsule_6_10.rb
|
334
336
|
- definitions/scenarios/upgrade_to_capsule_6_10_z.rb
|
337
|
+
- definitions/scenarios/upgrade_to_capsule_6_11.rb
|
338
|
+
- definitions/scenarios/upgrade_to_capsule_6_11_z.rb
|
335
339
|
- definitions/scenarios/upgrade_to_capsule_6_8.rb
|
336
340
|
- definitions/scenarios/upgrade_to_capsule_6_8_z.rb
|
337
341
|
- definitions/scenarios/upgrade_to_capsule_6_9.rb
|
338
342
|
- definitions/scenarios/upgrade_to_capsule_6_9_z.rb
|
339
|
-
- definitions/scenarios/upgrade_to_capsule_7_0.rb
|
340
|
-
- definitions/scenarios/upgrade_to_capsule_7_0_z.rb
|
341
343
|
- definitions/scenarios/upgrade_to_satellite_6_10.rb
|
342
344
|
- definitions/scenarios/upgrade_to_satellite_6_10_z.rb
|
345
|
+
- definitions/scenarios/upgrade_to_satellite_6_11.rb
|
346
|
+
- definitions/scenarios/upgrade_to_satellite_6_11_z.rb
|
343
347
|
- definitions/scenarios/upgrade_to_satellite_6_2.rb
|
344
348
|
- definitions/scenarios/upgrade_to_satellite_6_2_z.rb
|
345
349
|
- definitions/scenarios/upgrade_to_satellite_6_3.rb
|
@@ -356,8 +360,6 @@ files:
|
|
356
360
|
- definitions/scenarios/upgrade_to_satellite_6_8_z.rb
|
357
361
|
- definitions/scenarios/upgrade_to_satellite_6_9.rb
|
358
362
|
- definitions/scenarios/upgrade_to_satellite_6_9_z.rb
|
359
|
-
- definitions/scenarios/upgrade_to_satellite_7_0.rb
|
360
|
-
- definitions/scenarios/upgrade_to_satellite_7_0_z.rb
|
361
363
|
- extras/foreman-maintain.sh
|
362
364
|
- extras/foreman_protector/foreman-protector.conf
|
363
365
|
- extras/foreman_protector/foreman-protector.py
|
@@ -389,6 +391,8 @@ files:
|
|
389
391
|
- lib/foreman_maintain/concerns/directory_marker.rb
|
390
392
|
- lib/foreman_maintain/concerns/downstream.rb
|
391
393
|
- lib/foreman_maintain/concerns/finders.rb
|
394
|
+
- lib/foreman_maintain/concerns/firewall/iptables_maintenance_mode.rb
|
395
|
+
- lib/foreman_maintain/concerns/firewall/nftables_maintenance_mode.rb
|
392
396
|
- lib/foreman_maintain/concerns/hammer.rb
|
393
397
|
- lib/foreman_maintain/concerns/logger.rb
|
394
398
|
- lib/foreman_maintain/concerns/metadata.rb
|
@@ -1,15 +0,0 @@
|
|
1
|
-
module Procedures::Iptables
|
2
|
-
class AddMaintenanceModeChain < ForemanMaintain::Procedure
|
3
|
-
metadata do
|
4
|
-
label :iptables_add_maintenance_mode_chain
|
5
|
-
for_feature :iptables
|
6
|
-
description 'Add maintenance_mode chain to iptables'
|
7
|
-
tags :pre_migrations, :maintenance_mode_on
|
8
|
-
after :sync_plans_disable
|
9
|
-
end
|
10
|
-
|
11
|
-
def run
|
12
|
-
feature(:iptables).add_maintenance_mode_chain
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
@@ -1,15 +0,0 @@
|
|
1
|
-
module Procedures::Iptables
|
2
|
-
class RemoveMaintenanceModeChain < ForemanMaintain::Procedure
|
3
|
-
metadata do
|
4
|
-
label :iptables_remove_maintenance_mode_chain
|
5
|
-
for_feature :iptables
|
6
|
-
description 'Remove maintenance_mode chain from iptables'
|
7
|
-
tags :post_migrations, :maintenance_mode_off
|
8
|
-
after :sync_plans_enable
|
9
|
-
end
|
10
|
-
|
11
|
-
def run
|
12
|
-
feature(:iptables).remove_maintenance_mode_chain
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|