gitlab-qa 5.14.1 → 6.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (25) hide show
  1. checksums.yaml +4 -4
  2. data/.gitlab-ci.yml +41 -2
  3. data/README.md +52 -5
  4. data/docs/what_tests_can_be_run.md +101 -5
  5. data/lib/gitlab/qa.rb +4 -1
  6. data/lib/gitlab/qa/component/base.rb +32 -11
  7. data/lib/gitlab/qa/component/specs.rb +43 -11
  8. data/lib/gitlab/qa/docker/command.rb +15 -3
  9. data/lib/gitlab/qa/docker/engine.rb +5 -1
  10. data/lib/gitlab/qa/docker/shellout.rb +2 -2
  11. data/lib/gitlab/qa/release.rb +9 -2
  12. data/lib/gitlab/qa/report/json_test_results.rb +21 -0
  13. data/lib/gitlab/qa/report/junit_test_results.rb +21 -0
  14. data/lib/gitlab/qa/report/results_in_issues.rb +22 -18
  15. data/lib/gitlab/qa/report/test_result.rb +54 -0
  16. data/lib/gitlab/qa/runner.rb +23 -15
  17. data/lib/gitlab/qa/runtime/env.rb +11 -0
  18. data/lib/gitlab/qa/scenario/test/instance/geo.rb +0 -3
  19. data/lib/gitlab/qa/scenario/test/integration/geo.rb +4 -2
  20. data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb +204 -0
  21. data/lib/gitlab/qa/scenario/test/integration/praefect.rb +20 -8
  22. data/lib/gitlab/qa/scenario/test/sanity/version.rb +1 -1
  23. data/lib/gitlab/qa/version.rb +1 -1
  24. metadata +6 -3
  25. data/lib/gitlab/qa/scenario/test/integration/gitaly_ha.rb +0 -166
data/lib/gitlab/qa/scenario/test/integration/geo.rb CHANGED
@@ -27,13 +27,14 @@ module Gitlab
27
27
  gitlab_rails['db_pool'] = 5;
28
28
  gitlab_rails['geo_node_name'] = '#{primary.name}';
29
29
  gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0'];
30
+ gitlab_rails['packages_enabled'] = true;
30
31
  postgresql['listen_address'] = '0.0.0.0';
31
32
  postgresql['max_replication_slots'] = 1;
32
33
  postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0'];
33
34
  postgresql['sql_user_password'] = 'e1d1469ec5f533651918b4567a3ed1ae';
34
35
  postgresql['trust_auth_cidr_addresses'] = ['0.0.0.0/0','0.0.0.0/0'];
35
36
  sidekiq['concurrency'] = 2;
36
- unicorn['worker_processes'] = 2;
37
+ puma['worker_processes'] = 2;
37
38
  OMNIBUS
38
39
  primary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
39
40
 
@@ -50,11 +51,12 @@ module Gitlab
50
51
  gitlab_rails['db_pool'] = 5;
51
52
  gitlab_rails['geo_node_name'] = '#{secondary.name}';
52
53
  gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0'];
54
+ gitlab_rails['packages_enabled'] = true;
53
55
  postgresql['listen_address'] = '0.0.0.0';
54
56
  postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0'];
55
57
  postgresql['sql_user_password'] = 'e1d1469ec5f533651918b4567a3ed1ae';
56
58
  sidekiq['concurrency'] = 2;
57
- unicorn['worker_processes'] = 2;
59
+ puma['worker_processes'] = 2;
58
60
  OMNIBUS
59
61
  secondary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
60
62
 
data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb ADDED
@@ -0,0 +1,204 @@
1
+ module Gitlab
2
+ module QA
3
+ module Scenario
4
+ module Test
5
+ module Integration
6
+ class GitalyCluster < Scenario::Template
7
+ attr_reader :gitlab_name, :spec_suite
8
+
9
+ def initialize
10
+ @gitlab_name = 'gitlab-gitaly-cluster'
11
+ @primary_node_name = 'gitaly1'
12
+ @secondary_node_name = 'gitaly2'
13
+ @tertiary_node_name = 'gitaly3'
14
+ @praefect_node_name = 'praefect'
15
+ @database = 'postgres'
16
+ @spec_suite = 'Test::Instance::All'
17
+ @network = 'test'
18
+ end
19
+
20
+ # rubocop:disable Metrics/AbcSize
21
+ def perform(release, *rspec_args)
22
+ gitaly_primary_node = gitaly(@primary_node_name, release)
23
+ gitaly_secondary_node = gitaly(@secondary_node_name, release)
24
+ gitaly_tertiary_node = gitaly(@tertiary_node_name, release)
25
+
26
+ sql_node = Component::PostgreSQL.new.tap do |sql|
27
+ sql.name = @database
28
+ sql.network = @network
29
+ sql.instance(skip_teardown: true) do
30
+ sql.run_psql '-d template1 -c "CREATE DATABASE praefect_production OWNER postgres"'
31
+ end
32
+ end
33
+
34
+ praefect_node = Component::Gitlab.new.tap do |praefect|
35
+ praefect.release = QA::Release.new(release)
36
+ praefect.name = @praefect_node_name
37
+ praefect.network = @network
38
+ praefect.skip_availability_check = true
39
+
40
+ praefect.omnibus_config = praefect_omnibus_configuration
41
+
42
+ praefect.instance(skip_teardown: true)
43
+ end
44
+
45
+ Component::Gitlab.perform do |gitlab|
46
+ gitlab.release = QA::Release.new(release)
47
+ gitlab.name = gitlab_name
48
+ gitlab.network = @network
49
+
50
+ gitlab.omnibus_config = gitlab_omnibus_configuration
51
+ gitlab.instance do
52
+ puts "Running Gitaly Cluster specs!"
53
+
54
+ rspec_args << "--" unless rspec_args.include?('--')
55
+ rspec_args << %w[--tag gitaly_cluster]
56
+
57
+ Component::Specs.perform do |specs|
58
+ specs.suite = spec_suite
59
+ specs.release = gitlab.release
60
+ specs.network = gitlab.network
61
+ specs.args = [gitlab.address, *rspec_args]
62
+ end
63
+ end
64
+ end
65
+ ensure
66
+ praefect_node&.teardown
67
+ sql_node&.teardown
68
+ gitaly_primary_node&.teardown
69
+ gitaly_secondary_node&.teardown
70
+ gitaly_tertiary_node&.teardown
71
+ end
72
+ # rubocop:enable Metrics/AbcSize
73
+
74
+ private
75
+
76
+ def disable_other_services
77
+ <<~OMNIBUS
78
+ postgresql['enable'] = false;
79
+ redis['enable'] = false;
80
+ nginx['enable'] = false;
81
+ prometheus['enable'] = false;
82
+ grafana['enable'] = false;
83
+ puma['enable'] = false;
84
+ sidekiq['enable'] = false;
85
+ gitlab_workhorse['enable'] = false;
86
+ gitlab_rails['rake_cache_clear'] = false;
87
+ gitlab_rails['auto_migrate'] = false;
88
+ OMNIBUS
89
+ end
90
+
91
+ def praefect_omnibus_configuration
92
+ <<~OMNIBUS
93
+ #{disable_other_services}
94
+ gitaly['enable'] = false;
95
+ praefect['enable'] = true;
96
+ praefect['listen_addr'] = '0.0.0.0:2305';
97
+ praefect['prometheus_listen_addr'] = '0.0.0.0:9652';
98
+ praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN';
99
+ praefect['database_host'] = '#{@database}.#{@network}';
100
+ praefect['database_user'] = 'postgres';
101
+ praefect['database_port'] = 5432;
102
+ praefect['database_password'] = 'SQL_PASSWORD';
103
+ praefect['database_dbname'] = 'praefect_production';
104
+ praefect['database_sslmode'] = 'disable';
105
+ praefect['postgres_queue_enabled'] = true;
106
+ praefect['failover_enabled'] = true;
107
+ praefect['virtual_storages'] = {
108
+ 'default' => {
109
+ '#{@primary_node_name}' => {
110
+ 'address' => 'tcp://#{@primary_node_name}.#{@network}:8075',
111
+ 'token' => 'PRAEFECT_INTERNAL_TOKEN',
112
+ 'primary' => true
113
+ },
114
+ '#{@secondary_node_name}' => {
115
+ 'address' => 'tcp://#{@secondary_node_name}.#{@network}:8075',
116
+ 'token' => 'PRAEFECT_INTERNAL_TOKEN'
117
+ },
118
+ '#{@tertiary_node_name}' => {
119
+ 'address' => 'tcp://#{@tertiary_node_name}.#{@network}:8075',
120
+ 'token' => 'PRAEFECT_INTERNAL_TOKEN'
121
+ }
122
+ }
123
+ };
124
+ OMNIBUS
125
+ end
126
+
127
+ def gitaly_omnibus_configuration
128
+ <<~OMNIBUS
129
+ #{disable_other_services}
130
+ prometheus['enable'] = true;
131
+ prometheus_monitoring['enable'] = false;
132
+ gitaly['enable'] = true;
133
+ gitaly['listen_addr'] = '0.0.0.0:8075';
134
+ gitaly['prometheus_listen_addr'] = '0.0.0.0:9236';
135
+ gitaly['auth_token'] = 'PRAEFECT_INTERNAL_TOKEN';
136
+ gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
137
+ gitlab_rails['internal_api_url'] = 'http://#{@gitlab_name}.#{@network}';
138
+ git_data_dirs({
139
+ '#{@primary_node_name}' => {
140
+ 'path' => '/var/opt/gitlab/git-data'
141
+ },
142
+ '#{@secondary_node_name}' => {
143
+ 'path' => '/var/opt/gitlab/git-data'
144
+ },
145
+ '#{@tertiary_node_name}' => {
146
+ 'path' => '/var/opt/gitlab/git-data'
147
+ }
148
+ });
149
+ OMNIBUS
150
+ end
151
+
152
+ def gitlab_omnibus_configuration
153
+ <<~OMNIBUS
154
+ external_url 'http://#{@gitlab_name}.#{@network}';
155
+
156
+ git_data_dirs({
157
+ 'default' => {
158
+ 'gitaly_address' => 'tcp://#{@praefect_node_name}.#{@network}:2305',
159
+ 'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN'
160
+ }
161
+ });
162
+ gitaly['listen_addr'] = '0.0.0.0:8075';
163
+ gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
164
+ prometheus['scrape_configs'] = [
165
+ {
166
+ 'job_name' => 'praefect',
167
+ 'static_configs' => [
168
+ 'targets' => [
169
+ '#{@praefect_node_name}.#{@network}:9652'
170
+ ]
171
+ ]
172
+ },
173
+ {
174
+ 'job_name' => 'praefect-gitaly',
175
+ 'static_configs' => [
176
+ 'targets' => [
177
+ '#{@primary_node_name}.#{@network}:9236',
178
+ '#{@secondary_node_name}.#{@network}:9236',
179
+ '#{@tertiary_node_name}.#{@network}:9236'
180
+ ]
181
+ ]
182
+ }
183
+ ];
184
+ grafana['disable_login_form'] = false;
185
+ grafana['admin_password'] = 'GRAFANA_ADMIN_PASSWORD';
186
+ OMNIBUS
187
+ end
188
+
189
+ def gitaly(name, release)
190
+ Component::Gitlab.new.tap do |gitaly|
191
+ gitaly.release = QA::Release.new(release)
192
+ gitaly.name = name
193
+ gitaly.network = @network
194
+ gitaly.skip_availability_check = true
195
+ gitaly.omnibus_config = gitaly_omnibus_configuration
196
+ gitaly.instance(skip_teardown: true)
197
+ end
198
+ end
199
+ end
200
+ end
201
+ end
202
+ end
203
+ end
204
+ end
data/lib/gitlab/qa/scenario/test/integration/praefect.rb CHANGED
@@ -13,7 +13,11 @@ module Gitlab
13
13
  gitlab.name = 'gitlab'
14
14
  gitlab.network = 'test'
15
15
  gitlab.volumes = volumes
16
- gitlab.exec_commands = ['gitlab-psql -d template1 -c "CREATE DATABASE praefect_production OWNER gitlab"']
16
+ gitlab.exec_commands = [
17
+ 'gitlab-psql -d template1 -c "CREATE DATABASE praefect_production OWNER gitlab"',
18
+ 'mkdir -p /var/opt/gitlab/git-data/repositories/praefect',
19
+ 'chown -R git:root /var/opt/gitlab/git-data/repositories'
20
+ ]
17
21
 
18
22
  gitlab.act do
19
23
  prepare
@@ -21,7 +25,7 @@ module Gitlab
21
25
  reconfigure
22
26
  process_exec_commands
23
27
  wait
24
- teardown
28
+ teardown!
25
29
  end
26
30
  end
27
31
 
@@ -61,21 +65,26 @@ module Gitlab
61
65
  def omnibus_config_with_praefect
62
66
  <<~OMNIBUS
63
67
  gitaly['enable'] = true;
64
- gitaly['auth_token'] = 'praefect-gitaly-token';
68
+ gitaly['listen_addr'] = '0.0.0.0:8075';
69
+ gitaly['auth_token'] = 'secret-token';
65
70
  gitaly['storage'] = [
66
71
  {
67
72
  'name' => 'praefect-gitaly-0',
68
- 'path' => '/var/opt/gitlab/git-data/repositories'
73
+ 'path' => '/var/opt/gitlab/git-data/repositories/praefect'
74
+ },
75
+ {
76
+ 'name' => 'gitaly',
77
+ 'path' => '/var/opt/gitlab/git-data/repositories/gitaly'
69
78
  }
70
79
  ];
71
80
  praefect['enable'] = true;
72
81
  praefect['listen_addr'] = '0.0.0.0:2305';
73
- praefect['auth_token'] = 'praefect-token';
82
+ praefect['auth_token'] = 'secret-token';
74
83
  praefect['virtual_storages'] = {
75
84
  'default' => {
76
85
  'praefect-gitaly-0' => {
77
- 'address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket',
78
- 'token' => 'praefect-gitaly-token',
86
+ 'address' => 'tcp://localhost:8075',
87
+ 'token' => 'secret-token',
79
88
  'primary' => true
80
89
  }
81
90
  }
@@ -84,10 +93,13 @@ module Gitlab
84
93
  praefect['database_user'] = 'gitlab';
85
94
  praefect['database_dbname'] = 'praefect_production';
86
95
  praefect['postgres_queue_enabled'] = true;
87
- gitlab_rails['gitaly_token'] = 'praefect-token';
96
+ gitlab_rails['gitaly_token'] = 'secret-token';
88
97
  git_data_dirs({
89
98
  'default' => {
90
99
  'gitaly_address' => 'tcp://localhost:2305'
100
+ },
101
+ 'gitaly' => {
102
+ 'path' => '/var/opt/gitlab/git-data/repositories/gitaly'
91
103
  }
92
104
  });
93
105
  OMNIBUS
data/lib/gitlab/qa/scenario/test/sanity/version.rb CHANGED
@@ -24,7 +24,7 @@ module Gitlab
24
24
  end
25
25
  end
26
26
 
27
- project = "gitlab-org/#{QA::Release.new(release).project_name}"
27
+ project = "gitlab-org/#{QA::Release.new(release).api_project_name}"
28
28
  commit = recent_commits(project).find { |c| c['id'] == version }
29
29
 
30
30
  if commit
data/lib/gitlab/qa/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  module Gitlab
2
2
  module QA
3
- VERSION = '5.14.1'.freeze
3
+ VERSION = '6.0.0'.freeze
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-qa
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.14.1
4
+ version: 6.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Grzegorz Bizon
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-06-15 00:00:00.000000000 Z
11
+ date: 2020-08-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: climate_control
@@ -257,9 +257,12 @@ files:
257
257
  - lib/gitlab/qa/docker/shellout.rb
258
258
  - lib/gitlab/qa/docker/volumes.rb
259
259
  - lib/gitlab/qa/release.rb
260
+ - lib/gitlab/qa/report/json_test_results.rb
261
+ - lib/gitlab/qa/report/junit_test_results.rb
260
262
  - lib/gitlab/qa/report/prepare_stage_reports.rb
261
263
  - lib/gitlab/qa/report/results_in_issues.rb
262
264
  - lib/gitlab/qa/report/summary_table.rb
265
+ - lib/gitlab/qa/report/test_result.rb
263
266
  - lib/gitlab/qa/report/update_screenshot_path.rb
264
267
  - lib/gitlab/qa/reporter.rb
265
268
  - lib/gitlab/qa/runner.rb
@@ -284,7 +287,7 @@ files:
284
287
  - lib/gitlab/qa/scenario/test/instance/staging_geo.rb
285
288
  - lib/gitlab/qa/scenario/test/integration/elasticsearch.rb
286
289
  - lib/gitlab/qa/scenario/test/integration/geo.rb
287
- - lib/gitlab/qa/scenario/test/integration/gitaly_ha.rb
290
+ - lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb
288
291
  - lib/gitlab/qa/scenario/test/integration/group_saml.rb
289
292
  - lib/gitlab/qa/scenario/test/integration/instance_saml.rb
290
293
  - lib/gitlab/qa/scenario/test/integration/jira.rb
data/lib/gitlab/qa/scenario/test/integration/gitaly_ha.rb DELETED
@@ -1,166 +0,0 @@
1
- module Gitlab
2
- module QA
3
- module Scenario
4
- module Test
5
- module Integration
6
- class GitalyHA < Scenario::Template
7
- attr_reader :gitlab_name, :spec_suite
8
-
9
- def initialize
10
- @gitlab_name = 'gitlab-gitaly-ha'
11
- @primary_node = 'gitaly1'
12
- @secondary_node = 'gitaly2'
13
- @praefect_node = 'praefect'
14
- @database = 'postgres'
15
- @spec_suite = 'Test::Integration::GitalyHA'
16
- @network = 'test'
17
- end
18
-
19
- # rubocop:disable Metrics/AbcSize
20
- def perform(release, *rspec_args)
21
- gitaly(@primary_node, release) do
22
- gitaly(@secondary_node, release) do
23
- Component::PostgreSQL.perform do |sql|
24
- sql.name = @database
25
- sql.network = @network
26
-
27
- sql.instance do
28
- sql.run_psql '-d template1 -c "CREATE DATABASE praefect_production OWNER postgres"'
29
-
30
- Component::Gitlab.perform do |praefect|
31
- praefect.release = QA::Release.new(release)
32
- praefect.name = @praefect_node
33
- praefect.network = @network
34
- praefect.skip_availability_check = true
35
-
36
- praefect.omnibus_config = praefect_omnibus_configuration
37
-
38
- praefect.instance do
39
- Component::Gitlab.perform do |gitlab|
40
- gitlab.release = QA::Release.new(release)
41
- gitlab.name = gitlab_name
42
- gitlab.network = @network
43
-
44
- gitlab.omnibus_config = gitlab_omnibus_configuration
45
- gitlab.instance do
46
- puts "Running Gitaly HA specs!"
47
-
48
- Component::Specs.perform do |specs|
49
- specs.suite = spec_suite
50
- specs.release = gitlab.release
51
- specs.network = gitlab.network
52
- specs.args = [gitlab.address, *rspec_args]
53
- end
54
- end
55
- end
56
- end
57
- end
58
- end
59
- end
60
- end
61
- end
62
- end
63
- # rubocop:enable Metrics/AbcSize
64
-
65
- private
66
-
67
- def disable_other_services
68
- <<~OMNIBUS
69
- postgresql['enable'] = false;
70
- redis['enable'] = false;
71
- nginx['enable'] = false;
72
- prometheus['enable'] = false;
73
- grafana['enable'] = false;
74
- unicorn['enable'] = false;
75
- sidekiq['enable'] = false;
76
- gitlab_workhorse['enable'] = false;
77
- gitlab_rails['rake_cache_clear'] = false;
78
- gitlab_rails['auto_migrate'] = false;
79
- OMNIBUS
80
- end
81
-
82
- def praefect_omnibus_configuration
83
- <<~OMNIBUS
84
- #{disable_other_services}
85
- gitaly['enable'] = false;
86
- praefect['enable'] = true;
87
- praefect['listen_addr'] = '0.0.0.0:2305';
88
- praefect['prometheus_listen_addr'] = '0.0.0.0:9652';
89
- praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN';
90
- praefect['database_host'] = '#{@database}.#{@network}';
91
- praefect['database_user'] = 'postgres';
92
- praefect['database_port'] = 5432;
93
- praefect['database_password'] = 'SQL_PASSWORD';
94
- praefect['database_dbname'] = 'praefect_production';
95
- praefect['database_sslmode'] = 'disable';
96
- praefect['postgres_queue_enabled'] = true;
97
- praefect['failover_enabled'] = true;
98
- praefect['virtual_storages'] = {
99
- 'default' => {
100
- '#{@primary_node}' => {
101
- 'address' => 'tcp://#{@primary_node}.#{@network}:8075',
102
- 'token' => 'PRAEFECT_INTERNAL_TOKEN',
103
- 'primary' => true
104
- },
105
- '#{@secondary_node}' => {
106
- 'address' => 'tcp://#{@secondary_node}.#{@network}:8075',
107
- 'token' => 'PRAEFECT_INTERNAL_TOKEN'
108
- }
109
- }
110
- };
111
- OMNIBUS
112
- end
113
-
114
- def gitaly_omnibus_configuration
115
- <<~OMNIBUS
116
- #{disable_other_services}
117
- prometheus_monitoring['enable'] = false;
118
- gitaly['enable'] = true;
119
- gitaly['listen_addr'] = '0.0.0.0:8075';
120
- gitaly['prometheus_listen_addr'] = '0.0.0.0:9236';
121
- gitaly['auth_token'] = 'PRAEFECT_INTERNAL_TOKEN';
122
- gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
123
- gitlab_rails['internal_api_url'] = 'http://#{@gitlab_name}.#{@network}';
124
- git_data_dirs({
125
- '#{@primary_node}' => {
126
- 'path' => '/var/opt/gitlab/git-data'
127
- },
128
- '#{@secondary_node}' => {
129
- 'path' => '/var/opt/gitlab/git-data'
130
- }
131
- });
132
- OMNIBUS
133
- end
134
-
135
- def gitlab_omnibus_configuration
136
- <<~OMNIBUS
137
- git_data_dirs({
138
- 'default' => {
139
- 'gitaly_address' => 'tcp://#{@praefect_node}.#{@network}:2305',
140
- 'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN'
141
- }
142
- });
143
- gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
144
- OMNIBUS
145
- end
146
-
147
- def gitaly(name, release)
148
- Component::Gitlab.perform do |gitaly|
149
- gitaly.release = QA::Release.new(release)
150
- gitaly.name = name
151
- gitaly.network = @network
152
- gitaly.skip_availability_check = true
153
-
154
- gitaly.omnibus_config = gitaly_omnibus_configuration
155
-
156
- gitaly.instance do
157
- yield self
158
- end
159
- end
160
- end
161
- end
162
- end
163
- end
164
- end
165
- end
166
- end