bosh_aws_cpi 1.2579.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/aws_cpi +28 -0
- data/lib/cloud/aws/availability_zone_selector.rb +5 -7
- data/lib/cloud/aws/cloud.rb +203 -85
- data/lib/cloud/aws/helpers.rb +21 -3
- data/lib/cloud/aws/instance.rb +90 -0
- data/lib/cloud/aws/instance_manager.rb +274 -175
- data/lib/cloud/aws/instances_create_presenter.rb +29 -0
- data/lib/cloud/aws/resource_wait.rb +29 -14
- data/lib/cloud/aws/spot_manager.rb +129 -0
- data/lib/cloud/aws/stemcell_creator.rb +42 -16
- data/lib/cloud/aws/volume_properties.rb +44 -0
- data/lib/cloud/aws/volumes_create_presenter.rb +22 -0
- data/lib/cloud/aws.rb +5 -1
- data/scripts/stemcell-copy.sh +2 -2
- data/scripts/vendor_gems +5 -0
- metadata +34 -47
- data/README.md +0 -109
- data/lib/cloud/aws/version.rb +0 -5
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: a4dfbd1f01aaac029d31232f4e138d4ffa0cc558
|
|
4
|
+
data.tar.gz: a79bb1b483ca6095f9854aa708967ec3b44689f0
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 49c9096c1c5af778e1ff031f74baa9fef2d3f3ba222b78619221e18b55baf8ec1caddfbcca7ba438ee238c0e4dd4c507b3897eea7accf6228859b5bca54e8010
|
|
7
|
+
data.tar.gz: 2a658d7740f91eded63c62d34137e8cf3ebc4edde0bbaf6cfb5b8f1b8b185cfe8a4f1928c49013e610aac34bb0aadd500a09f6612e37e11375fab3e86719566b
|
data/bin/aws_cpi
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'yaml'
|
|
4
|
+
require 'ostruct'
|
|
5
|
+
require 'bosh/cpi'
|
|
6
|
+
require 'bundler'
|
|
7
|
+
|
|
8
|
+
Bundler.setup(:default)
|
|
9
|
+
require 'cloud/aws'
|
|
10
|
+
|
|
11
|
+
cpi_config = YAML.load_file(ARGV.shift)
|
|
12
|
+
cloud_config = OpenStruct.new(:logger => Logger.new(STDERR))
|
|
13
|
+
|
|
14
|
+
Bosh::Clouds::Config.configure(cloud_config)
|
|
15
|
+
|
|
16
|
+
cpi_log = StringIO.new # Empty on AWS
|
|
17
|
+
|
|
18
|
+
cpi_lambda = lambda do
|
|
19
|
+
unless cpi_config.has_key?('cloud') && cpi_config['cloud'].has_key?('properties')
|
|
20
|
+
raise "Could not find cloud properties in the configuration"
|
|
21
|
+
end
|
|
22
|
+
cloud_properties = cpi_config['cloud']['properties']
|
|
23
|
+
Bosh::Clouds::Aws.new(cloud_properties)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
cli = Bosh::Cpi::Cli.new(cpi_lambda, cpi_log, STDOUT)
|
|
27
|
+
|
|
28
|
+
cli.run(ARGF.read)
|
|
@@ -9,7 +9,10 @@ module Bosh::AwsCloud
|
|
|
9
9
|
|
|
10
10
|
def common_availability_zone(volume_az_names, resource_pool_az_name, vpc_subnet_az_name)
|
|
11
11
|
zone_names = (volume_az_names + [resource_pool_az_name, vpc_subnet_az_name]).compact.uniq
|
|
12
|
-
|
|
12
|
+
if zone_names.size > 1
|
|
13
|
+
volume_az_error_string = ", and volume in #{volume_az_names.join(', ')}" unless volume_az_names.empty?
|
|
14
|
+
raise Bosh::Clouds::CloudError, "can't use multiple availability zones: subnet in #{vpc_subnet_az_name}, VM in #{resource_pool_az_name}#{volume_az_error_string}"
|
|
15
|
+
end
|
|
13
16
|
|
|
14
17
|
zone_names.first || @default
|
|
15
18
|
end
|
|
@@ -31,10 +34,5 @@ module Bosh::AwsCloud
|
|
|
31
34
|
region.availability_zones.each { |zone| zones << zone.name }
|
|
32
35
|
zones[Random.rand(zones.size)]
|
|
33
36
|
end
|
|
34
|
-
|
|
35
|
-
def ensure_same_availability_zone(zone_names)
|
|
36
|
-
raise Bosh::Clouds::CloudError,
|
|
37
|
-
"can't use multiple availability zones: #{zone_names.join(', ')}" if zone_names.size > 1
|
|
38
|
-
end
|
|
39
37
|
end
|
|
40
|
-
end
|
|
38
|
+
end
|
data/lib/cloud/aws/cloud.rb
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
# Copyright (c) 2009-2012 VMware, Inc.
|
|
2
1
|
require 'cloud/aws/stemcell_finder'
|
|
3
2
|
|
|
4
3
|
module Bosh::AwsCloud
|
|
5
|
-
|
|
6
4
|
class Cloud < Bosh::Cloud
|
|
7
5
|
include Helpers
|
|
8
6
|
|
|
@@ -25,12 +23,50 @@ module Bosh::AwsCloud
|
|
|
25
23
|
def initialize(options)
|
|
26
24
|
@options = options.dup.freeze
|
|
27
25
|
validate_options
|
|
26
|
+
validate_credentials_source
|
|
28
27
|
|
|
29
28
|
@logger = Bosh::Clouds::Config.logger
|
|
29
|
+
aws_logger = @logger
|
|
30
|
+
|
|
31
|
+
@aws_params = {
|
|
32
|
+
credentials_source: aws_properties['credentials_source'] || 'static',
|
|
33
|
+
region: aws_properties['region'],
|
|
34
|
+
max_retries: aws_properties['max_retries'] || DEFAULT_MAX_RETRIES,
|
|
35
|
+
logger: aws_logger
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
%w(
|
|
39
|
+
http_read_timeout
|
|
40
|
+
http_wire_trace
|
|
41
|
+
proxy_uri
|
|
42
|
+
ssl_verify_peer
|
|
43
|
+
ssl_ca_file
|
|
44
|
+
ssl_ca_path
|
|
45
|
+
).each do |k|
|
|
46
|
+
@aws_params[k.to_sym] = aws_properties[k] unless aws_properties[k].nil?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# credentials_source could be static (default) or env_or_profile
|
|
50
|
+
# static credentials must be included in aws_properties
|
|
51
|
+
# env_or_profile credentials will use the AWS DefaultCredentialsProvider
|
|
52
|
+
# to find AWS credentials in environment variables or EC2 instance profiles
|
|
53
|
+
|
|
54
|
+
if @aws_params[:credentials_source] == 'static'
|
|
55
|
+
@aws_params[:access_key_id] = aws_properties['access_key_id']
|
|
56
|
+
@aws_params[:secret_access_key] = aws_properties['secret_access_key']
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# AWS Ruby SDK is threadsafe but Ruby autoload isn't,
|
|
60
|
+
# so we need to trigger eager autoload while constructing CPI
|
|
61
|
+
AWS.eager_autoload!
|
|
30
62
|
|
|
31
63
|
initialize_aws
|
|
32
64
|
initialize_registry
|
|
33
65
|
|
|
66
|
+
elb = AWS::ELB.new(@aws_params)
|
|
67
|
+
|
|
68
|
+
@instance_manager = InstanceManager.new(region, registry, elb, az_selector, @logger)
|
|
69
|
+
|
|
34
70
|
@metadata_lock = Mutex.new
|
|
35
71
|
end
|
|
36
72
|
|
|
@@ -84,39 +120,38 @@ module Bosh::AwsCloud
|
|
|
84
120
|
stemcell = StemcellFinder.find_by_region_and_id(region, stemcell_id)
|
|
85
121
|
|
|
86
122
|
begin
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
123
|
+
instance, block_device_agent_info = @instance_manager.create(
|
|
124
|
+
agent_id,
|
|
125
|
+
stemcell.image_id,
|
|
126
|
+
resource_pool,
|
|
127
|
+
network_spec,
|
|
128
|
+
(disk_locality || []),
|
|
129
|
+
environment,
|
|
130
|
+
options,
|
|
131
|
+
)
|
|
90
132
|
|
|
91
133
|
logger.info("Creating new instance '#{instance.id}'")
|
|
92
134
|
|
|
93
135
|
NetworkConfigurator.new(network_spec).configure(region, instance)
|
|
94
136
|
|
|
95
137
|
registry_settings = initial_agent_settings(
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
138
|
+
agent_id,
|
|
139
|
+
network_spec,
|
|
140
|
+
environment,
|
|
141
|
+
stemcell.root_device_name,
|
|
142
|
+
block_device_agent_info
|
|
100
143
|
)
|
|
101
144
|
registry.update_settings(instance.id, registry_settings)
|
|
102
145
|
|
|
103
146
|
instance.id
|
|
104
147
|
rescue => e # is this rescuing too much?
|
|
105
148
|
logger.error(%Q[Failed to create instance: #{e.message}\n#{e.backtrace.join("\n")}])
|
|
106
|
-
|
|
149
|
+
instance.terminate(fast_path_delete?) if instance
|
|
107
150
|
raise e
|
|
108
151
|
end
|
|
109
152
|
end
|
|
110
153
|
end
|
|
111
154
|
|
|
112
|
-
def default_ec2_endpoint
|
|
113
|
-
['ec2', aws_region, 'amazonaws.com'].compact.join('.')
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
def default_elb_endpoint
|
|
117
|
-
['elasticloadbalancing', aws_region, 'amazonaws.com'].compact.join('.')
|
|
118
|
-
end
|
|
119
|
-
|
|
120
155
|
##
|
|
121
156
|
# Delete EC2 instance ("terminate" in AWS language) and wait until
|
|
122
157
|
# it reports as terminated
|
|
@@ -124,7 +159,7 @@ module Bosh::AwsCloud
|
|
|
124
159
|
def delete_vm(instance_id)
|
|
125
160
|
with_thread_name("delete_vm(#{instance_id})") do
|
|
126
161
|
logger.info("Deleting instance '#{instance_id}'")
|
|
127
|
-
|
|
162
|
+
@instance_manager.find(instance_id).terminate(fast_path_delete?)
|
|
128
163
|
end
|
|
129
164
|
end
|
|
130
165
|
|
|
@@ -133,7 +168,7 @@ module Bosh::AwsCloud
|
|
|
133
168
|
# @param [String] instance_id EC2 instance id
|
|
134
169
|
def reboot_vm(instance_id)
|
|
135
170
|
with_thread_name("reboot_vm(#{instance_id})") do
|
|
136
|
-
|
|
171
|
+
@instance_manager.find(instance_id).reboot
|
|
137
172
|
end
|
|
138
173
|
end
|
|
139
174
|
|
|
@@ -142,7 +177,7 @@ module Bosh::AwsCloud
|
|
|
142
177
|
# @param [String] instance_id EC2 instance id
|
|
143
178
|
def has_vm?(instance_id)
|
|
144
179
|
with_thread_name("has_vm?(#{instance_id})") do
|
|
145
|
-
|
|
180
|
+
@instance_manager.find(instance_id).exists?
|
|
146
181
|
end
|
|
147
182
|
end
|
|
148
183
|
|
|
@@ -152,13 +187,21 @@ module Bosh::AwsCloud
|
|
|
152
187
|
# @param [optional, String] instance_id EC2 instance id
|
|
153
188
|
# of the VM that this disk will be attached to
|
|
154
189
|
# @return [String] created EBS volume id
|
|
155
|
-
def create_disk(size, instance_id = nil)
|
|
190
|
+
def create_disk(size, cloud_properties, instance_id = nil)
|
|
191
|
+
raise ArgumentError, 'disk size needs to be an integer' unless size.kind_of?(Integer)
|
|
156
192
|
with_thread_name("create_disk(#{size}, #{instance_id})") do
|
|
157
|
-
|
|
193
|
+
volume_properties = VolumeProperties.new(
|
|
194
|
+
size: size,
|
|
195
|
+
type: cloud_properties['type'],
|
|
196
|
+
iops: cloud_properties['iops'],
|
|
197
|
+
az: @az_selector.select_availability_zone(instance_id),
|
|
198
|
+
encrypted: cloud_properties['encrypted']
|
|
199
|
+
)
|
|
200
|
+
volume_properties.validate!
|
|
158
201
|
|
|
159
202
|
# if the disk is created for an instance, use the same availability zone as they must match
|
|
160
|
-
|
|
161
|
-
|
|
203
|
+
with_volume_options = VolumesCreatePresenter.new(volume_properties).present
|
|
204
|
+
volume = @ec2.volumes.create(with_volume_options)
|
|
162
205
|
|
|
163
206
|
logger.info("Creating volume '#{volume.id}'")
|
|
164
207
|
ResourceWait.for_volume(volume: volume, state: :available)
|
|
@@ -167,11 +210,16 @@ module Bosh::AwsCloud
|
|
|
167
210
|
end
|
|
168
211
|
end
|
|
169
212
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
213
|
+
##
|
|
214
|
+
# Check whether an EBS volume exists or not
|
|
215
|
+
#
|
|
216
|
+
# @param [String] disk_id EBS volume UUID
|
|
217
|
+
# @return [bool] whether the specific disk is there or not
|
|
218
|
+
def has_disk?(disk_id)
|
|
219
|
+
with_thread_name("has_disk?(#{disk_id})") do
|
|
220
|
+
@logger.info("Check the presence of disk with id `#{disk_id}'...")
|
|
221
|
+
@ec2.volumes[disk_id].exists?
|
|
222
|
+
end
|
|
175
223
|
end
|
|
176
224
|
|
|
177
225
|
##
|
|
@@ -184,15 +232,27 @@ module Bosh::AwsCloud
|
|
|
184
232
|
|
|
185
233
|
logger.info("Deleting volume `#{volume.id}'")
|
|
186
234
|
|
|
187
|
-
|
|
188
|
-
|
|
235
|
+
# Retry 1, 6, 11, 15, 15, 15.. seconds. The total time is ~10 min.
|
|
236
|
+
# VolumeInUse can be returned by AWS if disk was attached to VM
|
|
237
|
+
# that was recently removed.
|
|
238
|
+
tries = ResourceWait::DEFAULT_WAIT_ATTEMPTS
|
|
239
|
+
sleep_cb = ResourceWait.sleep_callback(
|
|
240
|
+
"Waiting for volume `#{volume.id}' to be deleted",
|
|
241
|
+
{ interval: 5, total: tries }
|
|
242
|
+
)
|
|
189
243
|
ensure_cb = Proc.new do |retries|
|
|
190
244
|
cloud_error("Timed out waiting to delete volume `#{volume.id}'") if retries == tries
|
|
191
245
|
end
|
|
192
|
-
|
|
246
|
+
errors = [AWS::EC2::Errors::VolumeInUse, AWS::EC2::Errors::RequestLimitExceeded]
|
|
247
|
+
|
|
248
|
+
Bosh::Common.retryable(tries: tries, sleep: sleep_cb, on: errors, ensure: ensure_cb) do
|
|
249
|
+
begin
|
|
250
|
+
volume.delete
|
|
251
|
+
rescue AWS::EC2::Errors::InvalidVolume::NotFound => e
|
|
252
|
+
logger.warn("Failed to delete disk '#{disk_id}' because it was not found: #{e.inspect}")
|
|
253
|
+
raise Bosh::Clouds::DiskNotFound.new(false), "Disk '#{disk_id}' not found"
|
|
254
|
+
end
|
|
193
255
|
|
|
194
|
-
Bosh::Common.retryable(tries: tries, sleep: sleep_cb, on: error, ensure: ensure_cb) do
|
|
195
|
-
volume.delete
|
|
196
256
|
true # return true to only retry on Exceptions
|
|
197
257
|
end
|
|
198
258
|
|
|
@@ -230,6 +290,9 @@ module Bosh::AwsCloud
|
|
|
230
290
|
end
|
|
231
291
|
logger.info("Attached `#{disk_id}' to `#{instance_id}'")
|
|
232
292
|
end
|
|
293
|
+
|
|
294
|
+
# log registry settings for debugging
|
|
295
|
+
logger.debug("updated registry settings: #{registry.read_settings(instance_id)}")
|
|
233
296
|
end
|
|
234
297
|
|
|
235
298
|
# Detach an EBS volume from an EC2 instance
|
|
@@ -240,14 +303,18 @@ module Bosh::AwsCloud
|
|
|
240
303
|
instance = @ec2.instances[instance_id]
|
|
241
304
|
volume = @ec2.volumes[disk_id]
|
|
242
305
|
|
|
306
|
+
if volume.exists?
|
|
307
|
+
detach_ebs_volume(instance, volume)
|
|
308
|
+
else
|
|
309
|
+
@logger.info("Disk `#{disk_id}' not found while trying to detach it from vm `#{instance_id}'...")
|
|
310
|
+
end
|
|
311
|
+
|
|
243
312
|
update_agent_settings(instance) do |settings|
|
|
244
313
|
settings["disks"] ||= {}
|
|
245
314
|
settings["disks"]["persistent"] ||= {}
|
|
246
315
|
settings["disks"]["persistent"].delete(disk_id)
|
|
247
316
|
end
|
|
248
317
|
|
|
249
|
-
detach_ebs_volume(instance, volume)
|
|
250
|
-
|
|
251
318
|
logger.info("Detached `#{disk_id}' from `#{instance_id}'")
|
|
252
319
|
end
|
|
253
320
|
end
|
|
@@ -266,21 +333,23 @@ module Bosh::AwsCloud
|
|
|
266
333
|
# @param [String] disk_id disk id of the disk to take the snapshot of
|
|
267
334
|
# @return [String] snapshot id
|
|
268
335
|
def snapshot_disk(disk_id, metadata)
|
|
336
|
+
metadata = Hash[metadata.map{|key,value| [key.to_s, value] }]
|
|
337
|
+
|
|
269
338
|
with_thread_name("snapshot_disk(#{disk_id})") do
|
|
270
339
|
volume = @ec2.volumes[disk_id]
|
|
271
340
|
devices = []
|
|
272
341
|
volume.attachments.each {|attachment| devices << attachment.device}
|
|
273
342
|
|
|
274
|
-
name = [
|
|
343
|
+
name = ['deployment', 'job', 'index'].collect { |key| metadata[key] }
|
|
275
344
|
name << devices.first.split('/').last unless devices.empty?
|
|
276
345
|
|
|
277
346
|
snapshot = volume.create_snapshot(name.join('/'))
|
|
278
347
|
logger.info("snapshot '#{snapshot.id}' of volume '#{disk_id}' created")
|
|
279
348
|
|
|
280
|
-
[
|
|
349
|
+
['agent_id', 'instance_id', 'director_name', 'director_uuid'].each do |key|
|
|
281
350
|
TagManager.tag(snapshot, key, metadata[key])
|
|
282
351
|
end
|
|
283
|
-
TagManager.tag(snapshot,
|
|
352
|
+
TagManager.tag(snapshot, 'device', devices.first) unless devices.empty?
|
|
284
353
|
TagManager.tag(snapshot, 'Name', name.join('/'))
|
|
285
354
|
|
|
286
355
|
ResourceWait.for_snapshot(snapshot: snapshot, state: :completed)
|
|
@@ -322,7 +391,7 @@ module Bosh::AwsCloud
|
|
|
322
391
|
network_configurator.configure(@ec2, instance)
|
|
323
392
|
|
|
324
393
|
update_agent_settings(instance) do |settings|
|
|
325
|
-
settings["networks"] = network_spec
|
|
394
|
+
settings["networks"] = agent_network_spec(network_spec)
|
|
326
395
|
end
|
|
327
396
|
end
|
|
328
397
|
end
|
|
@@ -332,7 +401,7 @@ module Bosh::AwsCloud
|
|
|
332
401
|
# we need to send the InstanceUpdater a request to do it for us
|
|
333
402
|
def compare_security_groups(instance, network_spec)
|
|
334
403
|
actual_group_names = instance.security_groups.collect { |sg| sg.name }
|
|
335
|
-
specified_group_names =
|
|
404
|
+
specified_group_names = extract_security_groups(network_spec)
|
|
336
405
|
if specified_group_names.empty?
|
|
337
406
|
new_group_names = Array(aws_properties["default_security_groups"])
|
|
338
407
|
else
|
|
@@ -387,13 +456,13 @@ module Bosh::AwsCloud
|
|
|
387
456
|
return creator.fake.id if creator.fake?
|
|
388
457
|
|
|
389
458
|
begin
|
|
390
|
-
# These
|
|
459
|
+
# These variables are used in 'ensure' clause
|
|
391
460
|
instance = nil
|
|
392
461
|
volume = nil
|
|
393
462
|
|
|
394
463
|
# 1. Create and mount new EBS volume (2GB default)
|
|
395
464
|
disk_size = stemcell_properties["disk"] || 2048
|
|
396
|
-
volume_id = create_disk(disk_size, current_vm_id)
|
|
465
|
+
volume_id = create_disk(disk_size, {}, current_vm_id)
|
|
397
466
|
volume = @ec2.volumes[volume_id]
|
|
398
467
|
instance = @ec2.instances[current_vm_id]
|
|
399
468
|
|
|
@@ -429,31 +498,33 @@ module Bosh::AwsCloud
|
|
|
429
498
|
# @param [Hash] metadata metadata key/value pairs
|
|
430
499
|
# @return [void]
|
|
431
500
|
def set_vm_metadata(vm, metadata)
|
|
501
|
+
metadata = Hash[metadata.map{|key,value| [key.to_s, value] }]
|
|
502
|
+
|
|
432
503
|
instance = @ec2.instances[vm]
|
|
433
504
|
|
|
434
505
|
metadata.each_pair do |key, value|
|
|
435
|
-
TagManager.tag(instance, key, value)
|
|
506
|
+
TagManager.tag(instance, key, value) unless key == 'name'
|
|
436
507
|
end
|
|
437
508
|
|
|
438
|
-
|
|
439
|
-
|
|
509
|
+
name = metadata['name']
|
|
510
|
+
if name
|
|
511
|
+
TagManager.tag(instance, "Name", name)
|
|
512
|
+
return
|
|
513
|
+
end
|
|
514
|
+
|
|
515
|
+
job = metadata['job']
|
|
516
|
+
index = metadata['index']
|
|
440
517
|
|
|
441
518
|
if job && index
|
|
442
519
|
name = "#{job}/#{index}"
|
|
443
|
-
elsif metadata[
|
|
444
|
-
name = "compiling/#{metadata[
|
|
520
|
+
elsif metadata['compiling']
|
|
521
|
+
name = "compiling/#{metadata['compiling']}"
|
|
445
522
|
end
|
|
446
523
|
TagManager.tag(instance, "Name", name) if name
|
|
447
524
|
rescue AWS::EC2::Errors::TagLimitExceeded => e
|
|
448
525
|
logger.error("could not tag #{instance.id}: #{e.message}")
|
|
449
526
|
end
|
|
450
527
|
|
|
451
|
-
# @note Not implemented in the AWS CPI
|
|
452
|
-
def validate_deployment(old_manifest, new_manifest)
|
|
453
|
-
# Not implemented in VSphere CPI as well
|
|
454
|
-
not_implemented(:validate_deployment)
|
|
455
|
-
end
|
|
456
|
-
|
|
457
528
|
def find_ebs_device(sd_name)
|
|
458
529
|
xvd_name = sd_name.gsub(/^\/dev\/sd/, "/dev/xvd")
|
|
459
530
|
|
|
@@ -469,7 +540,6 @@ module Bosh::AwsCloud
|
|
|
469
540
|
cloud_error("Cannot find EBS volume on current instance")
|
|
470
541
|
end
|
|
471
542
|
|
|
472
|
-
|
|
473
543
|
private
|
|
474
544
|
|
|
475
545
|
attr_reader :az_selector
|
|
@@ -484,7 +554,7 @@ module Bosh::AwsCloud
|
|
|
484
554
|
end
|
|
485
555
|
|
|
486
556
|
def aws_region
|
|
487
|
-
@aws_region ||= aws_properties.fetch('region'
|
|
557
|
+
@aws_region ||= aws_properties.fetch('region')
|
|
488
558
|
end
|
|
489
559
|
|
|
490
560
|
def fast_path_delete?
|
|
@@ -492,27 +562,16 @@ module Bosh::AwsCloud
|
|
|
492
562
|
end
|
|
493
563
|
|
|
494
564
|
def initialize_aws
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
aws_params[:proxy_uri] = aws_properties['proxy_uri'] if aws_properties['proxy_uri']
|
|
506
|
-
|
|
507
|
-
# AWS Ruby SDK is threadsafe but Ruby autoload isn't,
|
|
508
|
-
# so we need to trigger eager autoload while constructing CPI
|
|
509
|
-
AWS.eager_autoload!
|
|
510
|
-
|
|
511
|
-
AWS.config(aws_params)
|
|
512
|
-
|
|
513
|
-
@ec2 = AWS::EC2.new
|
|
514
|
-
@region = @ec2.regions[aws_region]
|
|
515
|
-
@az_selector = AvailabilityZoneSelector.new(@region, aws_properties['default_availability_zone'])
|
|
565
|
+
@ec2 = AWS::EC2.new(@aws_params)
|
|
566
|
+
# @ec2.regions[] does not properly set the endpoint on the region (bug in /aws/ec2/region_collection.rb)
|
|
567
|
+
# It just returns a Region object with nothing set but the name.
|
|
568
|
+
# As a workaround use the 'each' method, which is implemented correctly
|
|
569
|
+
begin
|
|
570
|
+
@region = @ec2.regions.select {|r| r.name == aws_region}.first
|
|
571
|
+
@az_selector = AvailabilityZoneSelector.new(@region, aws_properties['default_availability_zone'])
|
|
572
|
+
rescue Net::OpenTimeout => e
|
|
573
|
+
cloud_error("Please make sure the CPI has proper network access to AWS. #{e.inspect}")
|
|
574
|
+
end
|
|
516
575
|
end
|
|
517
576
|
|
|
518
577
|
def initialize_registry
|
|
@@ -548,11 +607,35 @@ module Bosh::AwsCloud
|
|
|
548
607
|
# AWS might still lie and say that the disk isn't ready yet, so
|
|
549
608
|
# we try again just to be really sure it is telling the truth
|
|
550
609
|
attachment = nil
|
|
551
|
-
|
|
610
|
+
|
|
611
|
+
logger.debug("Attaching '#{volume.id}' to '#{instance.id}' as '#{device_name}'")
|
|
612
|
+
|
|
613
|
+
# Retry every 1 sec for 15 sec, then every 15 sec for ~10 min
|
|
614
|
+
# VolumeInUse can be returned by AWS if disk was attached to VM
|
|
615
|
+
# that was recently removed.
|
|
616
|
+
tries = ResourceWait::DEFAULT_WAIT_ATTEMPTS
|
|
617
|
+
sleep_cb = ResourceWait.sleep_callback(
|
|
618
|
+
"Attaching volume `#{volume.id}' to #{instance.id}",
|
|
619
|
+
{ interval: 0, tries_before_max: 15, total: tries }
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
Bosh::Common.retryable(
|
|
623
|
+
on: [
|
|
624
|
+
AWS::EC2::Errors::IncorrectState,
|
|
625
|
+
AWS::EC2::Errors::VolumeInUse,
|
|
626
|
+
AWS::EC2::Errors::RequestLimitExceeded
|
|
627
|
+
],
|
|
628
|
+
sleep: sleep_cb,
|
|
629
|
+
tries: tries
|
|
630
|
+
) do |retries, error|
|
|
631
|
+
# Continue to retry after 15 attempts only for VolumeInUse
|
|
632
|
+
if retries > 15 && error.instance_of?(AWS::EC2::Errors::IncorrectState)
|
|
633
|
+
cloud_error("Failed to attach disk: #{error.message}")
|
|
634
|
+
end
|
|
635
|
+
|
|
552
636
|
attachment = volume.attach_to(instance, device_name)
|
|
553
637
|
end
|
|
554
638
|
|
|
555
|
-
logger.info("Attaching '#{volume.id}' to '#{instance.id}' as '#{device_name}'")
|
|
556
639
|
ResourceWait.for_attachment(attachment: attachment, state: :attached)
|
|
557
640
|
|
|
558
641
|
device_name = attachment.device
|
|
@@ -582,7 +665,6 @@ module Bosh::AwsCloud
|
|
|
582
665
|
hash[attachment.volume.id] = device_name
|
|
583
666
|
hash
|
|
584
667
|
end
|
|
585
|
-
|
|
586
668
|
if device_map[volume.id].nil?
|
|
587
669
|
raise Bosh::Clouds::DiskNotAttached.new(true),
|
|
588
670
|
"Disk `#{volume.id}' is not attached to instance `#{instance.id}'"
|
|
@@ -600,7 +682,7 @@ module Bosh::AwsCloud
|
|
|
600
682
|
#
|
|
601
683
|
def validate_options
|
|
602
684
|
required_keys = {
|
|
603
|
-
"aws" => ["
|
|
685
|
+
"aws" => ["region", "default_key_name"],
|
|
604
686
|
"registry" => ["endpoint", "user", "password"],
|
|
605
687
|
}
|
|
606
688
|
|
|
@@ -617,6 +699,30 @@ module Bosh::AwsCloud
|
|
|
617
699
|
raise ArgumentError, "missing configuration parameters > #{missing_keys.join(', ')}" unless missing_keys.empty?
|
|
618
700
|
end
|
|
619
701
|
|
|
702
|
+
##
|
|
703
|
+
# Checks AWS credentials settings to see if the CPI
|
|
704
|
+
# will be able to authenticate to AWS.
|
|
705
|
+
#
|
|
706
|
+
def validate_credentials_source
|
|
707
|
+
credentials_source = options['aws']['credentials_source'] || 'static'
|
|
708
|
+
|
|
709
|
+
if credentials_source != 'env_or_profile' && credentials_source != 'static'
|
|
710
|
+
raise ArgumentError, "Unknown credentials_source #{credentials_source}"
|
|
711
|
+
end
|
|
712
|
+
|
|
713
|
+
if credentials_source == 'static'
|
|
714
|
+
if options['aws']['access_key_id'].nil? || options['aws']['secret_access_key'].nil?
|
|
715
|
+
raise ArgumentError, "Must use access_key_id and secret_access_key with static credentials_source"
|
|
716
|
+
end
|
|
717
|
+
end
|
|
718
|
+
|
|
719
|
+
if credentials_source == 'env_or_profile'
|
|
720
|
+
if !options['aws']['access_key_id'].nil? || !options['aws']['secret_access_key'].nil?
|
|
721
|
+
raise ArgumentError, "Can't use access_key_id and secret_access_key with env_or_profile credentials_source"
|
|
722
|
+
end
|
|
723
|
+
end
|
|
724
|
+
end
|
|
725
|
+
|
|
620
726
|
# Generates initial agent settings. These settings will be read by agent
|
|
621
727
|
# from AWS registry (also a BOSH component) on a target instance. Disk
|
|
622
728
|
# conventions for amazon are:
|
|
@@ -630,23 +736,35 @@ module Bosh::AwsCloud
|
|
|
630
736
|
# @param [Hash] network_spec Agent network spec
|
|
631
737
|
# @param [Hash] environment
|
|
632
738
|
# @param [String] root_device_name root device, e.g. /dev/sda1
|
|
739
|
+
# @param [Hash] block_device_agent_info disk attachment information to merge into the disks section.
|
|
740
|
+
# keys are device type ("ephemeral", "raw_ephemeral") and values are array of strings representing the
|
|
741
|
+
# path to the block device. It is expected that "ephemeral" has exactly one value.
|
|
633
742
|
# @return [Hash]
|
|
634
|
-
def initial_agent_settings(agent_id, network_spec, environment, root_device_name)
|
|
743
|
+
def initial_agent_settings(agent_id, network_spec, environment, root_device_name, block_device_agent_info)
|
|
635
744
|
settings = {
|
|
636
745
|
"vm" => {
|
|
637
746
|
"name" => "vm-#{SecureRandom.uuid}"
|
|
638
747
|
},
|
|
639
748
|
"agent_id" => agent_id,
|
|
640
|
-
"networks" => network_spec,
|
|
749
|
+
"networks" => agent_network_spec(network_spec),
|
|
641
750
|
"disks" => {
|
|
642
751
|
"system" => root_device_name,
|
|
643
|
-
"ephemeral" => "/dev/sdb",
|
|
644
752
|
"persistent" => {}
|
|
645
753
|
}
|
|
646
754
|
}
|
|
647
755
|
|
|
756
|
+
settings["disks"].merge!(block_device_agent_info)
|
|
757
|
+
settings["disks"]["ephemeral"] = settings["disks"]["ephemeral"][0]["path"]
|
|
758
|
+
|
|
648
759
|
settings["env"] = environment if environment
|
|
649
760
|
settings.merge(agent_properties)
|
|
650
761
|
end
|
|
762
|
+
|
|
763
|
+
def agent_network_spec(network_spec)
|
|
764
|
+
Hash[*network_spec.map do |name, settings|
|
|
765
|
+
settings["use_dhcp"] = true
|
|
766
|
+
[name, settings]
|
|
767
|
+
end.flatten]
|
|
768
|
+
end
|
|
651
769
|
end
|
|
652
770
|
end
|
data/lib/cloud/aws/helpers.rb
CHANGED
|
@@ -1,8 +1,26 @@
|
|
|
1
|
-
# Copyright (c) 2009-2012 VMware, Inc.
|
|
2
|
-
|
|
3
1
|
module Bosh::AwsCloud
|
|
4
2
|
|
|
5
3
|
module Helpers
|
|
4
|
+
def default_ephemeral_disk_mapping
|
|
5
|
+
[
|
|
6
|
+
{
|
|
7
|
+
:device_name => '/dev/sdb',
|
|
8
|
+
:virtual_name => 'ephemeral0',
|
|
9
|
+
},
|
|
10
|
+
]
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def ebs_ephemeral_disk_mapping(volume_size_in_gb, volume_type, iops = nil)
|
|
14
|
+
ebs = {
|
|
15
|
+
volume_size: volume_size_in_gb,
|
|
16
|
+
volume_type: volume_type,
|
|
17
|
+
delete_on_termination: true,
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
ebs[:iops] = iops if iops
|
|
21
|
+
|
|
22
|
+
[{device_name: '/dev/sdb', ebs: ebs}]
|
|
23
|
+
end
|
|
6
24
|
|
|
7
25
|
##
|
|
8
26
|
# Raises CloudError exception
|
|
@@ -14,7 +32,7 @@ module Bosh::AwsCloud
|
|
|
14
32
|
raise Bosh::Clouds::CloudError, message
|
|
15
33
|
end
|
|
16
34
|
|
|
17
|
-
def
|
|
35
|
+
def extract_security_groups(networks_spec)
|
|
18
36
|
networks_spec.
|
|
19
37
|
values.
|
|
20
38
|
select { |network_spec| network_spec.has_key? "cloud_properties" }.
|