bosh_cli_plugin_micro 1.5.0.pre.1113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,555 @@
1
+ # Copyright (c) 2009-2012 VMware, Inc.
2
+
3
+ require 'open3'
4
+
5
+ module Bosh::Deployer
6
+
7
+ class DirectorGatewayError < RuntimeError; end
8
+
9
+ class InstanceManager
10
+
11
+ CONNECTION_EXCEPTIONS = [
12
+ Bosh::Agent::Error,
13
+ Errno::ECONNREFUSED,
14
+ Errno::ETIMEDOUT,
15
+ Bosh::Deployer::DirectorGatewayError,
16
+ HTTPClient::ConnectTimeoutError
17
+ ]
18
+
19
+ include Helpers
20
+
21
+ attr_reader :state
22
+ attr_accessor :renderer
23
+
24
+ class LoggerRenderer
25
+ attr_accessor :stage, :total, :index
26
+
27
+ def initialize
28
+ enter_stage("Deployer", 0)
29
+ end
30
+
31
+ def enter_stage(stage, total)
32
+ @stage = stage
33
+ @total = total
34
+ @index = 0
35
+ end
36
+
37
+ def update(state, task)
38
+ Config.logger.info("#{@stage} - #{state} #{task}")
39
+ @index += 1 if state == :finished
40
+ end
41
+ end
42
+
43
+ class << self
44
+
45
+ include Helpers
46
+
47
+ def create(config)
48
+ plugin = cloud_plugin(config)
49
+
50
+ begin
51
+ require "deployer/instance_manager/#{plugin}"
52
+ rescue LoadError
53
+ err "Could not find Provider Plugin: #{plugin}"
54
+ end
55
+ Bosh::Deployer::InstanceManager.const_get(plugin.capitalize).new(config)
56
+ end
57
+
58
+ end
59
+
60
+ def initialize(config)
61
+ Config.configure(config)
62
+
63
+ @state_yml = File.join(config["dir"], DEPLOYMENTS_FILE)
64
+ load_state(config["name"])
65
+
66
+ Config.uuid = state.uuid
67
+
68
+ @renderer = LoggerRenderer.new
69
+ end
70
+
71
+ def cloud
72
+ Config.cloud
73
+ end
74
+
75
+ def agent
76
+ Config.agent
77
+ end
78
+
79
+ def logger
80
+ Config.logger
81
+ end
82
+
83
+ def disk_model
84
+ nil
85
+ end
86
+
87
+ def instance_model
88
+ Models::Instance
89
+ end
90
+
91
+ def exists?
92
+ state.vm_cid != nil
93
+ end
94
+
95
+ def step(task)
96
+ renderer.update(:started, task)
97
+ result = yield
98
+ renderer.update(:finished, task)
99
+ result
100
+ end
101
+
102
+ def start
103
+ end
104
+
105
+ def stop
106
+ end
107
+
108
+ def with_lifecycle
109
+ start
110
+ yield
111
+ ensure
112
+ stop
113
+ end
114
+
115
+ def create_deployment(stemcell_tgz)
116
+ with_lifecycle do
117
+ create(stemcell_tgz)
118
+ end
119
+ end
120
+
121
+ def update_deployment(stemcell_tgz)
122
+ with_lifecycle do
123
+ update(stemcell_tgz)
124
+ end
125
+ end
126
+
127
+ def delete_deployment
128
+ with_lifecycle do
129
+ destroy
130
+ end
131
+ end
132
+
133
+ def create(stemcell_tgz)
134
+ err "VM #{state.vm_cid} already exists" if state.vm_cid
135
+ if state.stemcell_cid && state.stemcell_cid != state.stemcell_name
136
+ err "stemcell #{state.stemcell_cid} already exists"
137
+ end
138
+
139
+ renderer.enter_stage("Deploy Micro BOSH", 11)
140
+
141
+ state.stemcell_cid = create_stemcell(stemcell_tgz)
142
+ state.stemcell_name = File.basename(stemcell_tgz, ".tgz")
143
+ save_state
144
+
145
+ step "Creating VM from #{state.stemcell_cid}" do
146
+ state.vm_cid = create_vm(state.stemcell_cid)
147
+ update_vm_metadata(state.vm_cid, {"Name" => state.name})
148
+ discover_bosh_ip
149
+ end
150
+ save_state
151
+
152
+ step "Waiting for the agent" do
153
+ begin
154
+ wait_until_agent_ready
155
+ rescue *CONNECTION_EXCEPTIONS
156
+ err "Unable to connect to Bosh agent. Check logs for more details."
157
+ end
158
+ end
159
+
160
+ step "Updating persistent disk" do
161
+ update_persistent_disk
162
+ end
163
+
164
+ unless @apply_spec
165
+ step "Fetching apply spec" do
166
+ @apply_spec = Specification.new(agent.release_apply_spec)
167
+ end
168
+ end
169
+
170
+ apply
171
+
172
+ step "Waiting for the director" do
173
+ begin
174
+ wait_until_director_ready
175
+ rescue *CONNECTION_EXCEPTIONS
176
+ err "Unable to connect to Bosh Director. Retry manually or check logs for more details."
177
+ end
178
+ end
179
+ end
180
+
181
+ def destroy
182
+ renderer.enter_stage("Delete micro BOSH", 7)
183
+ agent_stop
184
+ if state.disk_cid
185
+ step "Deleting persistent disk `#{state.disk_cid}'" do
186
+ delete_disk(state.disk_cid, state.vm_cid)
187
+ state.disk_cid = nil
188
+ save_state
189
+ end
190
+ end
191
+ delete_vm
192
+ delete_stemcell
193
+ end
194
+
195
+ def update(stemcell_tgz)
196
+ renderer.enter_stage("Prepare for update", 5)
197
+ agent_stop
198
+ detach_disk(state.disk_cid)
199
+ delete_vm
200
+ # Do we always want to delete the stemcell?
201
+ # What if we are redeploying to the same stemcell version just so
202
+ # we can upgrade to a bigger persistent disk.
203
+ # Perhaps use "--preserve" to skip the delete?
204
+ delete_stemcell
205
+ create(stemcell_tgz)
206
+ end
207
+
208
+ def create_stemcell(stemcell_tgz)
209
+ unless is_tgz?(stemcell_tgz)
210
+ step "Using existing stemcell" do
211
+ end
212
+
213
+ return stemcell_tgz
214
+ end
215
+
216
+ Dir.mktmpdir("sc-") do |stemcell|
217
+ step "Unpacking stemcell" do
218
+ run_command("tar -zxf #{stemcell_tgz} -C #{stemcell}")
219
+ end
220
+
221
+ @apply_spec = Specification.load_from_stemcell(stemcell)
222
+
223
+ # load properties from stemcell manifest
224
+ properties = load_stemcell_manifest(stemcell)
225
+
226
+ # override with values from the deployment manifest
227
+ override = Config.cloud_options["properties"]["stemcell"]
228
+ properties["cloud_properties"].merge!(override) if override
229
+
230
+ step "Uploading stemcell" do
231
+ cloud.create_stemcell("#{stemcell}/image", properties["cloud_properties"])
232
+ end
233
+ end
234
+ rescue => e
235
+ logger.err("create stemcell failed: #{e.message}:\n#{e.backtrace.join("\n")}")
236
+ # make sure we clean up the stemcell if something goes wrong
237
+ delete_stemcell if is_tgz?(stemcell_tgz) && state.stemcell_cid
238
+ raise e
239
+ end
240
+
241
+ def create_vm(stemcell_cid)
242
+ resources = Config.resources['cloud_properties']
243
+ networks = Config.networks
244
+ env = Config.env
245
+ cloud.create_vm(state.uuid, stemcell_cid, resources, networks, nil, env)
246
+ end
247
+
248
+ def update_vm_metadata(vm, metadata)
249
+ cloud.set_vm_metadata(vm, metadata) if cloud.respond_to?(:set_vm_metadata)
250
+ rescue Bosh::Clouds::NotImplemented => e
251
+ logger.error(e)
252
+ end
253
+
254
+ def mount_disk(disk_cid)
255
+ step "Mount disk" do
256
+ agent.run_task(:mount_disk, disk_cid.to_s)
257
+ end
258
+ end
259
+
260
+ def unmount_disk(disk_cid)
261
+ step "Unmount disk" do
262
+ if disk_info.include?(disk_cid)
263
+ agent.run_task(:unmount_disk, disk_cid.to_s)
264
+ else
265
+ logger.error("not unmounting %s as it doesn't belong to me: %s" %
266
+ [disk_cid, disk_info])
267
+ end
268
+ end
269
+ end
270
+
271
+ def migrate_disk(src_disk_cid, dst_disk_cid)
272
+ step "Migrate disk" do
273
+ agent.run_task(:migrate_disk, src_disk_cid.to_s, dst_disk_cid.to_s)
274
+ end
275
+ end
276
+
277
+ def disk_info
278
+ return @disk_list if @disk_list
279
+ @disk_list = agent.list_disk
280
+ end
281
+
282
+ def create_disk
283
+ step "Create disk" do
284
+ size = Config.resources['persistent_disk']
285
+ state.disk_cid = cloud.create_disk(size, state.vm_cid)
286
+ save_state
287
+ end
288
+ end
289
+
290
+ # it is up to the caller to save/update disk state info
291
+ def delete_disk(disk_cid, vm_cid)
292
+ unmount_disk(disk_cid)
293
+
294
+ begin
295
+ step "Detach disk" do
296
+ cloud.detach_disk(vm_cid, disk_cid) if vm_cid
297
+ end
298
+ rescue Bosh::Clouds::DiskNotAttached
299
+ end
300
+
301
+ begin
302
+ step "Delete disk" do
303
+ cloud.delete_disk(disk_cid)
304
+ end
305
+ rescue Bosh::Clouds::DiskNotFound
306
+ end
307
+ end
308
+
309
+ # it is up to the caller to save/update disk state info
310
+ def attach_disk(disk_cid, is_create=false)
311
+ return unless disk_cid
312
+
313
+ cloud.attach_disk(state.vm_cid, disk_cid)
314
+ mount_disk(disk_cid)
315
+ end
316
+
317
+ def detach_disk(disk_cid)
318
+ unless disk_cid
319
+ err "Error: nil value given for persistent disk id"
320
+ end
321
+
322
+ unmount_disk(disk_cid)
323
+ step "Detach disk" do
324
+ cloud.detach_disk(state.vm_cid, disk_cid)
325
+ end
326
+ end
327
+
328
+ def attach_missing_disk
329
+ if state.disk_cid
330
+ attach_disk(state.disk_cid, true)
331
+ end
332
+ end
333
+
334
+ def check_persistent_disk
335
+ return if state.disk_cid.nil?
336
+ agent_disk_cid = disk_info.first
337
+ if agent_disk_cid != state.disk_cid
338
+ err "instance #{state.vm_cid} has invalid disk: " +
339
+ "Agent reports #{agent_disk_cid} while " +
340
+ "deployer's record shows #{state.disk_cid}"
341
+ end
342
+ end
343
+
344
+ def update_persistent_disk
345
+ attach_missing_disk
346
+ check_persistent_disk
347
+
348
+ if state.disk_cid.nil?
349
+ create_disk
350
+ attach_disk(state.disk_cid, true)
351
+ elsif persistent_disk_changed?
352
+ size = Config.resources['persistent_disk']
353
+
354
+ # save a reference to the old disk
355
+ old_disk_cid = state.disk_cid
356
+
357
+ # create a new disk and attach it
358
+ new_disk_cid = cloud.create_disk(size, state.vm_cid)
359
+ attach_disk(new_disk_cid, true)
360
+
361
+ # migrate data (which mounts the disks)
362
+ migrate_disk(old_disk_cid, new_disk_cid)
363
+
364
+ # replace the old with the new in the state file
365
+ state.disk_cid = new_disk_cid
366
+
367
+ # delete the old disk
368
+ delete_disk(old_disk_cid, state.vm_cid)
369
+ end
370
+ ensure
371
+ save_state
372
+ end
373
+
374
+ def apply(spec = nil)
375
+ agent_stop
376
+
377
+ spec ||= @apply_spec
378
+
379
+ step "Applying micro BOSH spec" do
380
+ # first update spec with infrastructure specific stuff
381
+ update_spec(spec)
382
+ # then update spec with generic changes
383
+ agent.run_task(:apply, spec.update(bosh_ip, service_ip))
384
+ end
385
+
386
+ agent_start
387
+ end
388
+
389
+ def discover_bosh_ip
390
+ bosh_ip
391
+ end
392
+
393
+ def service_ip
394
+ bosh_ip
395
+ end
396
+
397
+ def check_dependencies
398
+ # nothing to check, move on...
399
+ end
400
+
401
+ private
402
+
403
+ def bosh_ip
404
+ Config.bosh_ip
405
+ end
406
+
407
+ def agent_stop
408
+ step "Stopping agent services" do
409
+ begin
410
+ agent.run_task(:stop)
411
+ rescue
412
+ end
413
+ end
414
+ end
415
+
416
+ def agent_start
417
+ step "Starting agent services" do
418
+ agent.run_task(:start)
419
+ end
420
+ end
421
+
422
+ def wait_until_ready(component, wait_time = 1, retries = 300)
423
+ Bosh::Common.retryable(sleep: wait_time, tries: retries, on: CONNECTION_EXCEPTIONS) do |tries, e|
424
+ logger.debug("Waiting for #{component} to be ready: #{e.inspect}") if tries > 0
425
+ yield
426
+ true
427
+ end
428
+ end
429
+
430
+ def agent_port
431
+ uri = URI.parse(Config.cloud_options["properties"]["agent"]["mbus"])
432
+
433
+ uri.port
434
+ end
435
+
436
+ def wait_until_agent_ready #XXX >> agent_client
437
+ remote_tunnel(@registry_port)
438
+
439
+ wait_until_ready("agent") { agent.ping }
440
+ end
441
+
442
+ def wait_until_director_ready
443
+ port = @apply_spec.director_port
444
+ url = "https://#{bosh_ip}:#{port}/info"
445
+
446
+ wait_until_ready("director", 1, 600) do
447
+
448
+ http_client = HTTPClient.new
449
+
450
+ http_client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
451
+ http_client.ssl_config.verify_callback = Proc.new {}
452
+
453
+ response = http_client.get(url)
454
+ message = "Nginx has started but the application it is proxying to has not started yet."
455
+ raise Bosh::Deployer::DirectorGatewayError.new(message) if response.status == 502 || response.status == 503
456
+ info = Yajl::Parser.parse(response.body)
457
+ logger.info("Director is ready: #{info.inspect}")
458
+ end
459
+ end
460
+
461
+ def delete_stemcell
462
+ err "Cannot find existing stemcell" unless state.stemcell_cid
463
+
464
+ if state.stemcell_cid == state.stemcell_name
465
+ step "Preserving stemcell" do
466
+ end
467
+ else
468
+ step "Delete stemcell" do
469
+ cloud.delete_stemcell(state.stemcell_cid)
470
+ end
471
+ end
472
+
473
+ state.stemcell_cid = nil
474
+ state.stemcell_name = nil
475
+ save_state
476
+ end
477
+
478
+ def delete_vm
479
+ err "Cannot find existing VM" unless state.vm_cid
480
+
481
+ step "Delete VM" do
482
+ cloud.delete_vm(state.vm_cid)
483
+ end
484
+ state.vm_cid = nil
485
+ save_state
486
+ end
487
+
488
+ def load_deployments
489
+ if File.exists?(@state_yml)
490
+ logger.info("Loading existing deployment data from: #{@state_yml}")
491
+ Psych.load_file(@state_yml)
492
+ else
493
+ logger.info("No existing deployments found (will save to #{@state_yml})")
494
+ { "instances" => [], "disks" => [] }
495
+ end
496
+ end
497
+
498
+ def load_apply_spec(dir)
499
+ load_spec("#{dir}/apply_spec.yml") do
500
+ err "this isn't a micro bosh stemcell - apply_spec.yml missing"
501
+ end
502
+ end
503
+
504
+ def load_stemcell_manifest(dir)
505
+ load_spec("#{dir}/stemcell.MF") do
506
+ err "this isn't a stemcell - stemcell.MF missing"
507
+ end
508
+ end
509
+
510
+ def load_spec(file)
511
+ yield unless File.exist?(file)
512
+ logger.info("Loading yaml from #{file}")
513
+ Psych.load_file(file)
514
+ end
515
+
516
+ def generate_unique_name
517
+ SecureRandom.uuid
518
+ end
519
+
520
+ def load_state(name)
521
+ @deployments = load_deployments
522
+
523
+ disk_model.insert_multiple(@deployments["disks"]) if disk_model
524
+ instance_model.insert_multiple(@deployments["instances"])
525
+
526
+ @state = instance_model.find(:name => name)
527
+ if @state.nil?
528
+ @state = instance_model.new
529
+ @state.uuid = "bm-#{generate_unique_name}"
530
+ @state.name = name
531
+ @state.save
532
+ else
533
+ discover_bosh_ip
534
+ end
535
+ end
536
+
537
+ def save_state
538
+ state.save
539
+ @deployments["instances"] = instance_model.map { |instance| instance.values }
540
+ @deployments["disks"] = disk_model.map { |disk| disk.values } if disk_model
541
+
542
+ File.open(@state_yml, "w") do |file|
543
+ file.write(Psych.dump(@deployments))
544
+ end
545
+ end
546
+
547
+ def run_command(command)
548
+ output, status = Open3.capture2e(command)
549
+ if status.exitstatus != 0
550
+ $stderr.puts output
551
+ err "'#{command}' failed with exit status=#{status.exitstatus} [#{output}]"
552
+ end
553
+ end
554
+ end
555
+ end
@@ -0,0 +1,6 @@
1
+ # Copyright (c) 2009-2012 VMware, Inc.
2
+
3
+ module Bosh::Deployer::Models
4
+ class Instance < Sequel::Model(Bosh::Deployer::Config.db[:instances])
5
+ end
6
+ end
@@ -0,0 +1,97 @@
1
+ module Bosh::Deployer
2
+ class Specification
3
+
4
+ def self.load_from_stemcell(dir)
5
+ spec = load_apply_spec(dir)
6
+ Specification.new(spec)
7
+ end
8
+
9
+ def self.load_apply_spec(dir)
10
+ file = "apply_spec.yml"
11
+ apply_spec = File.join(dir, file)
12
+ unless File.exist?(apply_spec)
13
+ err "this isn't a micro bosh stemcell - #{file} missing"
14
+ end
15
+ Psych.load_file(apply_spec)
16
+ end
17
+
18
+ attr_accessor :spec
19
+ attr_accessor :properties
20
+
21
+ def initialize(spec)
22
+ @spec = spec
23
+ @properties = @spec["properties"]
24
+ end
25
+
26
+ # Update the spec with the IP of the micro bosh instance.
27
+ # @param [String] bosh_ip IP address of the micro bosh VM
28
+ # @param [String] service_ip private IP of the micro bosh VM on AWS/OS,
29
+ # or the same as the bosh_ip if vSphere/vCloud
30
+ def update(bosh_ip, service_ip)
31
+ # set the director name to what is specified in the micro_bosh.yml
32
+ if Config.name
33
+ @properties["director"] = {} unless @properties["director"]
34
+ @properties["director"]["name"] = Config.name
35
+ end
36
+
37
+ # on AWS blobstore and nats need to use an elastic IP (if available),
38
+ # as when the micro bosh instance is re-created during a deployment,
39
+ # it might get a new private IP
40
+ %w{blobstore nats}.each do |service|
41
+ update_agent_service_address(service, bosh_ip)
42
+ end
43
+
44
+ services = %w{director redis blobstore nats registry dns}
45
+ services.each do |service|
46
+ update_service_address(service, service_ip)
47
+ end
48
+
49
+ # health monitor does not listen to any ports, so there is no
50
+ # need to update the service address, but we still want to
51
+ # be able to override values in the apply_spec
52
+ override_property(@properties, "hm", Config.spec_properties["hm"])
53
+
54
+ override_property(@properties, "director", Config.spec_properties["director"])
55
+ set_property(@properties, "ntp", Config.spec_properties["ntp"])
56
+ set_property(@properties, "compiled_package_cache", Config.spec_properties["compiled_package_cache"])
57
+
58
+ @spec
59
+ end
60
+
61
+ # @param [String] name property name to delete from the spec
62
+ def delete(name)
63
+ @spec.delete(name)
64
+ end
65
+
66
+ # @return [String] the port the director runs on
67
+ def director_port
68
+ @properties["director"]["port"]
69
+ end
70
+
71
+ private
72
+
73
+ # update the agent service section from the contents of the apply_spec
74
+ def update_agent_service_address(service, address)
75
+ agent = @properties["agent"] ||= {}
76
+ svc = agent[service] ||= {}
77
+ svc["address"] = address
78
+
79
+ override_property(agent, service, Config.agent_properties[service])
80
+ end
81
+
82
+ def update_service_address(service, address)
83
+ return unless @properties[service]
84
+ @properties[service]["address"] = address
85
+
86
+ override_property(@properties, service, Config.spec_properties[service])
87
+ end
88
+
89
+ def set_property(properties, key, value)
90
+ properties[key] = value unless value.nil?
91
+ end
92
+
93
+ def override_property(properties, service, override)
94
+ properties[service].merge!(override) if override
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,7 @@
1
+ # Copyright (c) 2009-2012 VMware, Inc.
2
+
3
+ module Bosh
4
+ module Deployer
5
+ VERSION = '1.5.0.pre.1113'
6
+ end
7
+ end
data/lib/deployer.rb ADDED
@@ -0,0 +1,23 @@
1
+ # Copyright (c) 2009-2012 VMware, Inc.
2
+
3
+ module Bosh; module Deployer; end; end
4
+
5
+ require "agent_client"
6
+ require "fileutils"
7
+ require "forwardable"
8
+ require "sequel"
9
+ require "sequel/adapters/sqlite"
10
+ require "cloud"
11
+ require "logger"
12
+ require "tmpdir"
13
+ require "securerandom"
14
+ require "yaml"
15
+ require "yajl"
16
+ require "common/common"
17
+ require "common/thread_formatter"
18
+
19
+ require "deployer/version"
20
+ require "deployer/helpers"
21
+ require "deployer/config"
22
+ require "deployer/specification"
23
+ require "deployer/instance_manager"