cloudfoundry_blue_green_deploy 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ module CloudfoundryBlueGreenDeploy
2
+ class BlueGreenDeployError < StandardError; end
3
+ end
@@ -0,0 +1,90 @@
1
+ require_relative './command_line'
2
+ require_relative './route'
3
+ require_relative './app'
4
+
5
+ module CloudfoundryBlueGreenDeploy
6
+
7
+ class CloudfoundryCliError < StandardError; end
8
+
9
+ class Cloudfoundry
10
+
11
+ def self.apps
12
+ apps = []
13
+ cmd = "cf apps"
14
+ output = CommandLine.backtick(cmd)
15
+ found_header = false
16
+
17
+ lines = output.lines
18
+
19
+ lines.each do |line|
20
+ line = line.split
21
+ if line[0] == 'name' && found_header == false
22
+ found_header = true
23
+ next
24
+ end
25
+
26
+ if found_header
27
+ apps << App.new(name: line[0], state: line[1])
28
+ end
29
+ end
30
+
31
+ apps
32
+ end
33
+
34
+ def self.push(app)
35
+ execute("cf push #{app}")
36
+ end
37
+
38
+ def self.stop(app)
39
+ execute("cf stop #{app}")
40
+ end
41
+
42
+ def self.routes
43
+ routes = []
44
+ cmd = "cf routes"
45
+ output = CommandLine.backtick(cmd)
46
+ success = !output.include?('FAILED')
47
+ if success
48
+ lines = output.lines
49
+ found_header = false
50
+ lines.each do |line|
51
+ line = line.split
52
+ if line[0] == 'host' && found_header == false
53
+ found_header = true
54
+ next
55
+ end
56
+
57
+ if found_header
58
+ routes << Route.new(line[0], line[1], line[2])
59
+ end
60
+ end
61
+ routes
62
+ else
63
+ raise CloudfoundryCliError.new("\"#{cmd}\" returned \"#{success}\". The output of the command was \n\"#{output}\".")
64
+ end
65
+ end
66
+
67
+ def self.unmap_route(app, domain, host)
68
+ execute("cf unmap-route #{app} #{domain} -n #{host}")
69
+ end
70
+
71
+ def self.map_route(app, domain, host)
72
+ execute("cf map-route #{app} #{domain} -n #{host}")
73
+ end
74
+
75
+ private
76
+
77
+ def self.execute(cmd)
78
+ success = CommandLine.system(cmd)
79
+ handle_success_or_failure(cmd, success)
80
+ end
81
+
82
+ def self.handle_success_or_failure(cmd, success)
83
+ if success
84
+ return success
85
+ else
86
+ raise CloudfoundryCliError.new("\"#{cmd}\" returned \"#{success}\". Look for details in \"FAILED\" above.")
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,15 @@
1
+ module CloudfoundryBlueGreenDeploy
2
+ class CommandLine
3
+ DEBUG = false
4
+ def self.backtick(command)
5
+
6
+ output = `export CF_COLOR=false; #{command}`
7
+ puts "CommandLine.backtick(): \"#{output}\"" if DEBUG
8
+ output
9
+ end
10
+
11
+ def self.system(command)
12
+ Kernel.system(command)
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,11 @@
1
+ require 'rails'
2
+
3
+ module CloudfoundryBlueGreenDeploy
4
+ class Railtie < Rails::Railtie
5
+ railtie_name :cloudfoundry_blue_green_deploy
6
+
7
+ rake_tasks do
8
+ load 'cloudfoundry_blue_green_deploy/tasks/cf.rake'
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,10 @@
1
+ module CloudfoundryBlueGreenDeploy
2
+ class Route
3
+ attr_reader :host, :domain, :app
4
+ def initialize(host, domain, app)
5
+ @host = host
6
+ @domain = domain
7
+ @app = app == '' ? nil : app
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,28 @@
1
+ require 'cloudfoundry_blue_green_deploy'
2
+
3
+ namespace :cf do
4
+ desc 'Only run on the first application instance'
5
+ task :on_first_instance do
6
+ instance_index = JSON.parse(ENV['VCAP_APPLICATION'])['instance_index'] rescue nil
7
+ exit(0) unless instance_index == 0
8
+ end
9
+
10
+ desc 'Reroutes "live" traffic to specified app through provided URL'
11
+ task :blue_green_deploy, :web_app_name do |t, args|
12
+ web_app_name = args[:web_app_name]
13
+ worker_app_names = args.extras.to_a
14
+ if worker_app_names.last == 'with_shutter'
15
+ worker_app_names.pop
16
+ with_shutter = true
17
+ else
18
+ with_shutter = false
19
+ end
20
+
21
+ deploy_config = CloudfoundryBlueGreenDeploy::BlueGreenDeployConfig.new(load_manifest, web_app_name, worker_app_names, with_shutter)
22
+ CloudfoundryBlueGreenDeploy::BlueGreenDeploy.make_it_so(web_app_name, worker_app_names, deploy_config)
23
+ end
24
+
25
+ def load_manifest
26
+ YAML.load_file('manifest.yml')
27
+ end
28
+ end
@@ -0,0 +1,3 @@
1
+ module CloudfoundryBlueGreenDeploy
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,140 @@
1
+ require 'spec_helper'
2
+
3
+ module CloudfoundryBlueGreenDeploy
4
+ describe BlueGreenDeployConfig do
5
+ let(:cf_manifest) { YAML.load_file('spec/manifest.yml') }
6
+ let(:web_app_name) { 'the-web-app' }
7
+ let(:web_url_name) { 'the-web-url' }
8
+ let(:worker_app_names) { ['the-web-app-worker', 'hard-worker'] }
9
+ let(:with_shutter) { false }
10
+ let(:target_color) { nil }
11
+ let(:deploy_config) do
12
+ config = BlueGreenDeployConfig.new(cf_manifest, web_app_name, worker_app_names, with_shutter)
13
+ config.target_color = target_color
14
+ config
15
+ end
16
+
17
+ describe '#initialize' do
18
+ subject { deploy_config }
19
+ context 'given a parsed conforming manifest.yml' do
20
+ it 'calculates the "Hot URL"' do
21
+ expect(subject.hot_url).to eq "#{web_url_name}"
22
+ end
23
+
24
+ it 'determines the "domain" (i.e the Cloud Foundry domain)' do
25
+ expect(subject.domain).to eq 'cfapps.io'
26
+ end
27
+
28
+ context 'user requested "shutter treatment"' do
29
+ let(:with_shutter) { true }
30
+ it 'indicates "use shutter"' do
31
+ expect(subject.with_shutter).to eq true
32
+ end
33
+ end
34
+ end
35
+
36
+ describe '(vetting parameters against the contents of the manifest)' do
37
+ context 'given the "web_app_name" parameter does not match any of the applications defined in the manifest.yml' do
38
+ let(:web_app_name) { 'the-web-pap' }
39
+ it 'raises an InvalidManifestError' do
40
+ expect{subject}.to raise_error InvalidManifestError
41
+ end
42
+ end
43
+
44
+ context 'given one of the instances of a worker application is not defined in the manifest' do
45
+ let(:worker_app_names) { ['the-web-app-wroker', 'hard-worker'] }
46
+ it 'raises an InvalidManifestError' do
47
+ expect{subject}.to raise_error InvalidManifestError
48
+ end
49
+ end
50
+ end
51
+
52
+ context 'given the "web_app_name" matches, but the host name is not defined' do
53
+ let(:cf_manifest) { {"applications"=>[{"name"=>"the-web-app-blue"}]} }
54
+ let(:worker_app_names) { [] }
55
+ it 'raises an InvalidManifestError' do
56
+ expect{subject}.to raise_error InvalidManifestError
57
+ end
58
+ end
59
+
60
+ context 'given the "web_app_name" matches, but the domain is not defined' do
61
+ let(:cf_manifest) { {"applications"=>[{"name"=>"the-web-app-blue", "host"=> "#{web_url_name}"}]} }
62
+ let(:worker_app_names) { [] }
63
+ it 'raises an InvalidManifestError' do
64
+ expect{subject}.to raise_error InvalidManifestError
65
+ end
66
+ end
67
+
68
+ end
69
+
70
+ context 'the target color was calculated by Blue Green deploy' do
71
+ let(:target_color) { 'green' }
72
+
73
+ describe '.target_web_app_name' do
74
+ subject { deploy_config.target_web_app_name }
75
+ it 'calculates the "target" web app name' do
76
+ expect(subject).to eq "the-web-app-#{target_color}"
77
+ end
78
+ end
79
+
80
+ describe '.target_worker_app_names' do
81
+ subject { deploy_config.target_worker_app_names }
82
+
83
+ it 'calculates the "target" worker app names' do
84
+ expect(subject[0]).to eq 'the-web-app-worker-green'
85
+ expect(subject[1]).to eq 'hard-worker-green'
86
+ end
87
+ end
88
+ end
89
+
90
+ describe '.shutter_app_name' do
91
+ subject { deploy_config.shutter_app_name }
92
+ it 'provides the CF app name for the Shutter app.' do
93
+ expect(subject).to eq "#{web_app_name}-shutter"
94
+ end
95
+ end
96
+
97
+
98
+ describe '#strip_color' do
99
+ let(:app_name_with_color) { 'some-app-name-here-yay-blue' }
100
+ subject { BlueGreenDeployConfig.strip_color(app_name_with_color) }
101
+
102
+ it 'returns just the name of the app' do
103
+ expect(subject).to eq 'some-app-name-here-yay'
104
+ end
105
+
106
+ end
107
+
108
+ describe '#toggle_app_color' do
109
+ let(:app_name) { 'app_name' }
110
+ let(:target_app_name) { "#{app_name}-#{starting_color}" }
111
+ subject { BlueGreenDeployConfig.toggle_app_color(target_app_name) }
112
+
113
+ context 'where named app is the green instance' do
114
+ let(:starting_color) { 'green' }
115
+ it 'provides the blue app name' do
116
+ expect(subject).to eq "#{app_name}-blue"
117
+ end
118
+ end
119
+ end
120
+
121
+ describe '.is_in_target?' do
122
+ let(:target_color) { 'green' }
123
+ let(:app_name) { "app_name-#{app_color}" }
124
+ subject { deploy_config.is_in_target?(app_name) }
125
+
126
+ context 'when the specified app IS the name of the target app' do
127
+ let(:app_color) { target_color }
128
+ it 'returns true' do
129
+ expect(subject).to be true
130
+ end
131
+ end
132
+ context 'when the specified app is NOT the name of the target app' do
133
+ let(:app_color) { BlueGreenDeployConfig.toggle_color(target_color) }
134
+ it 'returns false' do
135
+ expect(subject).to be false
136
+ end
137
+ end
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,287 @@
1
+ require 'spec_helper'
2
+ require_relative 'cloudfoundry_fake'
3
+
4
+ module CloudfoundryBlueGreenDeploy
5
+ describe BlueGreenDeploy do
6
+ let(:cf_manifest) { YAML.load_file('spec/manifest.yml') }
7
+ let(:worker_app_names) { ['the-web-app-worker'] }
8
+ let(:deploy_config) { BlueGreenDeployConfig.new(cf_manifest, app_name, worker_app_names, with_shutter) }
9
+ let(:domain) { 'cfapps.io' }
10
+ let(:hot_url) { 'the-web-url' }
11
+ let(:app_name) { 'the-web-app' }
12
+ let(:with_shutter) { nil }
13
+
14
+ describe '#make_it_so' do
15
+ context 'steady-state deploy (not first deploy, already a hot app)' do
16
+ let(:worker_apps) { worker_app_names }
17
+ let(:target_color) { 'green' }
18
+ let(:current_hot_app) { 'blue' }
19
+
20
+ subject { BlueGreenDeploy.make_it_so(app_name, worker_apps, deploy_config) }
21
+
22
+ before do
23
+ allow(BlueGreenDeploy).to receive(:cf).and_return(CloudfoundryFake)
24
+ CloudfoundryFake.init_route_table(domain, app_name, hot_url, current_hot_app)
25
+ CloudfoundryFake.init_app_list_with_workers_for(app_name)
26
+ end
27
+
28
+ context 'AND deploy does not require shutter' do
29
+ let(:with_shutter) { false }
30
+ it 'instructs Cloud Foundry to deploy the specified web app; ' +
31
+ 'THEN, deploys each of the specified worker apps, stopping their counterparts; ' +
32
+ 'and THEN, makes the specified web app "hot" ' +
33
+ '(mapping the "hot" route to it and unmapping that "hot" route from it`s counterpart)' do
34
+ green_or_blue = BlueGreenDeployConfig.toggle_color(target_color)
35
+ old_worker_app_full_name = "#{worker_apps.first}-#{green_or_blue}"
36
+ new_worker_app_full_name = "#{worker_apps.first}-#{target_color}"
37
+ new_web_app_full_name = "#{app_name}-#{target_color}"
38
+ old_web_app_full_name = "#{app_name}-#{green_or_blue}"
39
+
40
+ expect(CloudfoundryFake).to receive(:push).with(new_web_app_full_name).ordered.and_call_original
41
+ expect(CloudfoundryFake).to receive(:push).with(new_worker_app_full_name).ordered.and_call_original
42
+ expect(CloudfoundryFake).to receive(:stop).with(old_worker_app_full_name).ordered.and_call_original
43
+ expect(CloudfoundryFake).to receive(:map_route).with(new_web_app_full_name, domain, hot_url).ordered.and_call_original
44
+ expect(CloudfoundryFake).to receive(:unmap_route).with(old_web_app_full_name, domain, hot_url).ordered.and_call_original
45
+
46
+ subject
47
+ end
48
+ end
49
+
50
+ context 'AND deploy requires shutter' do
51
+ let(:with_shutter) { true }
52
+ it 'instructs Cloud Foundry to deploy the specified web app; ' +
53
+ 'THEN, deploys each of the specified worker apps, stopping their counterparts; ' +
54
+ 'and THEN, makes the specified web app "hot" ' +
55
+ '(mapping the "hot" route to it and unmapping that "hot" route from it`s counterpart)' do
56
+ green_or_blue = BlueGreenDeployConfig.toggle_color(target_color)
57
+ shutter_app_name = deploy_config.shutter_app_name
58
+ old_worker_app_full_name = "#{worker_apps.first}-#{green_or_blue}"
59
+ new_worker_app_full_name = "#{worker_apps.first}-#{target_color}"
60
+ new_web_app_full_name = "#{app_name}-#{target_color}"
61
+ old_web_app_full_name = "#{app_name}-#{green_or_blue}"
62
+
63
+ expect(CloudfoundryFake).to receive(:push).with(shutter_app_name).ordered.and_call_original
64
+ expect(CloudfoundryFake).to receive(:map_route).with(shutter_app_name, domain, hot_url).ordered.and_call_original
65
+ expect(CloudfoundryFake).to receive(:unmap_route).with(old_web_app_full_name, domain, hot_url).ordered.and_call_original
66
+ expect(CloudfoundryFake).to receive(:push).with(new_web_app_full_name).ordered.and_call_original
67
+ expect(CloudfoundryFake).to receive(:push).with(new_worker_app_full_name).ordered.and_call_original
68
+ expect(CloudfoundryFake).to receive(:stop).with(old_worker_app_full_name).ordered.and_call_original
69
+ expect(CloudfoundryFake).to receive(:map_route).with(new_web_app_full_name, domain, hot_url).ordered.and_call_original
70
+ expect(CloudfoundryFake).to receive(:unmap_route).with(shutter_app_name, domain, hot_url).ordered.and_call_original
71
+
72
+ subject
73
+ end
74
+
75
+ end
76
+ end
77
+
78
+ context 'it is a first deploy' do
79
+ let(:target_color) { nil }
80
+ let(:worker_apps) { worker_app_names }
81
+ subject { BlueGreenDeploy.make_it_so(app_name, worker_apps, deploy_config) }
82
+ before do
83
+ allow(BlueGreenDeploy).to receive(:cf).and_return(CloudfoundryFake)
84
+ end
85
+
86
+ context 'there is no hot app' do
87
+ let(:current_hot_app) { nil }
88
+ before { CloudfoundryFake.clear_route_table }
89
+
90
+ context 'there are no hot worker apps' do
91
+ let(:worker_app_names) { [] }
92
+ before { CloudfoundryFake.clear_app_list }
93
+ it 'deploys "blue" instances' do
94
+ subject
95
+ hot_web_app = CloudfoundryFake.find_route(hot_url).app
96
+ expect(BlueGreenDeploy.get_color_stem(hot_web_app)).to eq 'blue'
97
+ CloudfoundryFake.started_apps.each do |worker_app|
98
+ expect(BlueGreenDeploy.get_color_stem(worker_app.name)).to eq 'blue'
99
+ end
100
+ end
101
+ end
102
+
103
+ context 'there ARE hot worker apps' do
104
+ before do
105
+ CloudfoundryFake.init_app_list_from_names(worker_app_names)
106
+ CloudfoundryFake.mark_app_as_started("#{worker_app_names.first}-blue")
107
+ end
108
+ it 'raises an InvalidRouteStateError' do
109
+ expect{ subject }.to raise_error(InvalidRouteStateError)
110
+ end
111
+ end
112
+ end
113
+ end
114
+ end
115
+
116
+ describe '#get_hot_worker_names' do
117
+ subject { BlueGreenDeploy.get_hot_worker_names }
118
+ let(:target_color) { 'green' }
119
+
120
+ context 'there are no started worker apps' do
121
+ before do
122
+ allow(BlueGreenDeploy).to receive(:cf).and_return(CloudfoundryFake)
123
+ CloudfoundryFake.init_app_list_from_names(worker_app_names)
124
+ end
125
+
126
+ it 'returns an empty array' do
127
+ expect(subject).to eq []
128
+ end
129
+ end
130
+
131
+ context 'a worker app is started (and another is stopped)' do
132
+ before do
133
+ allow(BlueGreenDeploy).to receive(:cf).and_return(CloudfoundryFake)
134
+ CloudfoundryFake.init_app_list_from_names(worker_app_names)
135
+ CloudfoundryFake.mark_workers_as_started(worker_app_names, target_color)
136
+ end
137
+
138
+ it 'returns just the started worker app' do
139
+ expect(subject).to eq(worker_app_names.map { |app_name| "#{app_name}-#{target_color}" })
140
+ end
141
+ end
142
+ end
143
+
144
+ describe '#ready_for_takeoff' do
145
+ let(:target_color) { 'green' }
146
+ subject { BlueGreenDeploy.ready_for_takeoff(hot_app_name, both_invalid_and_valid_hot_worker_names, deploy_config) }
147
+ before { allow(BlueGreenDeploy).to receive(:cf).and_return(CloudfoundryFake) }
148
+
149
+ context 'first deploy: there are no apps deployed' do
150
+ let(:hot_app_name) { nil }
151
+ let(:worker_app_names) { [] }
152
+ let(:both_invalid_and_valid_hot_worker_names) { worker_app_names }
153
+ before { CloudfoundryFake.init_app_list(worker_app_names) }
154
+
155
+ it 'allows the deploy to proceed' do
156
+ expect{ subject }.not_to raise_error
157
+ end
158
+ end
159
+
160
+ context 'in subsequent deploys' do
161
+ let(:hot_app_name) { "#{app_name}-#{current_hot_app}" }
162
+ let(:worker_apps) { CloudfoundryFake.apps }
163
+ let(:both_invalid_and_valid_hot_worker_names) { worker_apps.select { |app| app.state == 'started' }.map(&:name) }
164
+ before do
165
+ CloudfoundryFake.init_route_table(domain, app_name, hot_url, current_hot_app)
166
+ CloudfoundryFake.init_app_list_with_workers_for(app_name)
167
+ end
168
+ context 'the target color is the cold app color.' do
169
+ let(:current_hot_app) { 'blue' }
170
+ let(:target_color) { 'green' }
171
+ let(:deploy_config) do
172
+ config = BlueGreenDeployConfig.new(cf_manifest, app_name, worker_app_names, with_shutter)
173
+ config.target_color = target_color
174
+ config
175
+ end
176
+
177
+ it 'does not raise an error: "It`s kosh!"' do
178
+ expect{ subject }.to_not raise_error
179
+ end
180
+
181
+ context 'but, one or more of the target worker apps is already hot' do
182
+ before do
183
+ CloudfoundryFake.replace_app(App.new(name: "#{app_name}-worker-#{target_color}", state: 'started'))
184
+ both_invalid_and_valid_hot_worker_names = ["#{app_name}-worker-#{target_color}"]
185
+ end
186
+
187
+ it 'raises an InvalidWorkerStateError' do
188
+ expect{ subject }.to raise_error(InvalidWorkerStateError)
189
+ end
190
+ end
191
+ end
192
+
193
+
194
+ context 'and there is no current hot app and there are started worker apps' do
195
+ let(:target_color) { nil }
196
+ let(:current_hot_app) { '' }
197
+ let(:hot_app_name) { nil }
198
+ before do
199
+ CloudfoundryFake.remove_route(hot_url)
200
+ CloudfoundryFake.mark_app_as_started(CloudfoundryFake.apps.sample.name)
201
+ end
202
+
203
+ it 'raises an InvalidRouteStateError' do
204
+ expect{ subject }.to raise_error(InvalidRouteStateError)
205
+ end
206
+ end
207
+ end
208
+ end
209
+
210
+ describe '#get_hot_web_app' do
211
+ subject { BlueGreenDeploy.get_hot_web_app(hot_url) }
212
+ let(:current_hot_color) { 'green' }
213
+ let(:hot_app) { "#{app_name}-#{current_hot_color}" }
214
+
215
+ before do
216
+ allow(BlueGreenDeploy).to receive(:cf).and_return(CloudfoundryFake)
217
+ CloudfoundryFake.init_route_table(domain, app_name, hot_url, current_hot_color)
218
+ end
219
+
220
+ it 'returns the app mapped to that Host URL' do
221
+ expect(subject).to eq hot_app
222
+ end
223
+
224
+ context 'when there is no app mapped to the hot url' do
225
+ before { CloudfoundryFake.remove_route(hot_url) }
226
+
227
+ it 'returns nil' do
228
+ expect(subject).to be_nil
229
+ end
230
+ end
231
+ end
232
+
233
+
234
+ describe '#make_hot' do
235
+ let(:target_color) { 'blue' }
236
+ let(:current_hot_app) { 'green' }
237
+ let(:deploy_config) do
238
+ config = BlueGreenDeployConfig.new(cf_manifest, app_name, worker_app_names, with_shutter)
239
+ config.target_color = target_color
240
+ config
241
+ end
242
+ subject { BlueGreenDeploy.make_hot(app_name, deploy_config) }
243
+
244
+ before do
245
+ allow(BlueGreenDeploy).to receive(:cf).and_return(CloudfoundryFake)
246
+ CloudfoundryFake.init_route_table(domain, app_name, hot_url, current_hot_app)
247
+
248
+ end
249
+
250
+ context 'when there is no current hot app' do
251
+ before do
252
+ CloudfoundryFake.remove_route(hot_url)
253
+ end
254
+
255
+ it 'the target_color is mapped to the hot_url' do
256
+ subject
257
+ expect(BlueGreenDeploy.get_hot_web_app(hot_url)).to eq "#{app_name}-#{target_color}"
258
+ end
259
+ end
260
+
261
+ context 'when there IS a hot URL route, but it is not mapped to any app' do
262
+ before do
263
+ CloudfoundryFake.remove_route(hot_url)
264
+ CloudfoundryFake.add_route(Route.new(hot_url, domain, nil))
265
+
266
+ end
267
+
268
+ it 'the target_color is mapped to the hot_url' do
269
+ subject
270
+ expect(BlueGreenDeploy.get_hot_web_app(hot_url)).to eq "#{app_name}-#{target_color}"
271
+ end
272
+ end
273
+
274
+ context 'when the hot url IS mapped to an app, already' do
275
+ it 'the app that was mapped to the hot_url is no longer mapped to hot_url' do
276
+ subject
277
+ expect(BlueGreenDeploy.get_hot_web_app(hot_url)).to_not eq "#{app_name}-#{current_hot_app}"
278
+ end
279
+
280
+ it 'the target_color is mapped to the hot_url' do
281
+ subject
282
+ expect(BlueGreenDeploy.get_hot_web_app(hot_url)).to eq "#{app_name}-#{target_color}"
283
+ end
284
+ end
285
+ end
286
+ end
287
+ end