bosh_aws_cpi 2.0.1 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f1678fa3712efeb30ac293ada0fdb2183a3c043
-  data.tar.gz: eb18fe472d0f6f7fe0e0bedca3c64de37c3b2a55
+  metadata.gz: f772026cd4993ec84c960766136602f1a3e0ecb8
+  data.tar.gz: fa5a32aed55a00e0c51ff304fbf7d33d22883427
 SHA512:
-  metadata.gz: e0b8e1fbe36086aeaa4ac308ab30b5e2069c453b7722e5e2db1a0786e9fe4e608dc2cace7e4bc5ae82959f3fc245391954a54a356c5d42cc9237ccfc06fef368
-  data.tar.gz: 34e7d8ce2a7a05c13f9d6093d2391674358fcac953a268fa00f903575dad2d198b8cabb77e2f460a75e2ff8bc5dd71c87bca2f4134f6fb91230860f7edf983e7
+  metadata.gz: ded5cb96c6cfae851201917d64530825fc6b97440648422f0b8f7763c11d58d555b2b476fa65942e4d4b7acc2f3894e5f27fcc745cc852de4d551f45d4c63c42
+  data.tar.gz: 2f84f8df96570ad6b5e9147263879c658c9011d30e882693f0441c4b803600e83605853366b495cdd05d5c8b6a87ee476a0aee9eb7869f034e7e40e5326cf5e8

data/README.md

@@ -9,10 +9,6 @@ These options are passed to the AWS CPI when it is instantiated.
 
 ### AWS options
 
-* `access_key_id` (required)
-  AWS IAM user access key
-* `secret_access_key` (required)
-  AWS IAM secret access key
 * `default_key_name` (required)
   default AWS ssh key name to assign to created virtual machines
 * `default_security_groups` (required)
@@ -21,6 +17,14 @@ These options are passed to the AWS CPI when it is instantiated.
   local path to the ssh private key, must match `default_key_name`
 * `region` (required)
   EC2 region
+* `credentials_source` (optional)
+  Where to get AWS credentials. This can be set to `static` for to use an `access_key_id` and `secret_access_key` or `env_or_profile` to get the credentials from environment variables or an EC2 instance profile. Defaults to `static` if not set
+* `access_key_id` (optional, required when `credentials_source` is `static`)
+  AWS IAM user access key
+* `secret_access_key` (optional, required when `credentials_source` is `static`)
+  AWS IAM secret access key
+* `iam_instance_profile` (optional)
+  the [IAM Instance Profile](http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-usingrole-instanceprofile.html) to use for the instance. This allows EC2 instances to use [IAM Roles](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) when working with AWS APIs.
 * `ec2_endpoint` (optional)
   URL of the EC2 endpoint to connect to, defaults to the endpoint corresponding to the selected region,
   or `default_ec2_endpoint` if no region has been selected
@@ -57,6 +61,8 @@ These options are specified under `cloud_options` in the `resource_pools` sectio
   which [type of instance](http://aws.amazon.com/ec2/instance-types/) the VMs should belong to
 * `spot_bid_price` (optional)
   the [AWS spot instance](http://aws.amazon.com/ec2/purchasing-options/spot-instances/) bid price to use.  When specified spot instances are started rather than on demand instances.  _NB: this will dramatically slow down resource pool creation._
+* `iam_instance_profile` (optional)
+   the [IAM Instance Profile](http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-usingrole-instanceprofile.html) to use for the instance. This allows EC2 instances to use [IAM Roles](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) when working with AWS APIs.
 
 ### Network options
 

data/lib/cloud/aws/cloud.rb

@@ -25,13 +25,13 @@ module Bosh::AwsCloud
     def initialize(options)
       @options = options.dup.freeze
       validate_options
+      validate_credentials_source
 
       @logger = Bosh::Clouds::Config.logger
       aws_logger = @logger
 
       @aws_params = {
-        access_key_id:     aws_properties['access_key_id'],
-        secret_access_key: aws_properties['secret_access_key'],
+        credentials_source: aws_properties['credentials_source'] || 'static',
         region:            aws_properties['region'],
         ec2_endpoint:      aws_properties['ec2_endpoint'] || default_ec2_endpoint,
         elb_endpoint:      aws_properties['elb_endpoint'] || default_elb_endpoint,
@@ -50,6 +50,16 @@ module Bosh::AwsCloud
         @aws_params[k.to_sym] = aws_properties[k] unless aws_properties[k].nil?
       end
 
+      # credentials_source could be static (default) or env_or_profile
+      # static credentials must be included in aws_properties
+      # env_or_profile credentials will use the AWS DefaultCredentialsProvider
+      # to find AWS credentials in environment variables or EC2 instance profiles
+
+      if @aws_params[:credentials_source] == 'static'
+        @aws_params[:access_key_id] = aws_properties['access_key_id']
+        @aws_params[:secret_access_key] = aws_properties['secret_access_key']
+      end
+
       # AWS Ruby SDK is threadsafe but Ruby autoload isn't,
       # so we need to trigger eager autoload while constructing CPI
       AWS.eager_autoload!
@@ -57,13 +67,7 @@ module Bosh::AwsCloud
       initialize_aws
       initialize_registry
 
-      aws_elb_params = {
-        access_key_id:     aws_properties['access_key_id'],
-        secret_access_key: aws_properties['secret_access_key'],
-        region:            aws_properties['region']
-      }
-
-      elb = AWS::ELB.new(aws_elb_params)
+      elb = AWS::ELB.new(@aws_params)
 
       @instance_manager = InstanceManager.new(region, registry, elb, az_selector, @logger)
 
@@ -120,7 +124,7 @@ module Bosh::AwsCloud
         stemcell = StemcellFinder.find_by_region_and_id(region, stemcell_id)
 
         begin
-          instance = @instance_manager.create(
+          instance, block_device_agent_info = @instance_manager.create(
             agent_id,
             stemcell.image_id,
             resource_pool,
@@ -139,6 +143,7 @@ module Bosh::AwsCloud
             network_spec,
             environment,
             stemcell.root_device_name,
+            block_device_agent_info
           )
           registry.update_settings(instance.id, registry_settings)
 
@@ -674,7 +679,6 @@ module Bosh::AwsCloud
         hash[attachment.volume.id] = device_name
         hash
       end
-
       if device_map[volume.id].nil?
         raise Bosh::Clouds::DiskNotAttached.new(true),
               "Disk `#{volume.id}' is not attached to instance `#{instance.id}'"
@@ -692,7 +696,7 @@ module Bosh::AwsCloud
     #
     def validate_options
       required_keys = {
-          "aws" => ["access_key_id", "secret_access_key", "region", "default_key_name"],
+          "aws" => ["region", "default_key_name"],
           "registry" => ["endpoint", "user", "password"],
       }
 
@@ -709,6 +713,30 @@ module Bosh::AwsCloud
       raise ArgumentError, "missing configuration parameters > #{missing_keys.join(', ')}" unless missing_keys.empty?
     end
 
+    ##
+    # Checks AWS credentials settings to see if the CPI
+    # will be able to authenticate to AWS.
+    #
+    def validate_credentials_source
+      credentials_source = options['aws']['credentials_source'] || 'static'
+
+      if credentials_source != 'env_or_profile' && credentials_source != 'static'
+        raise ArgumentError, "Unknown credentials_source #{credentials_source}"
+      end
+
+      if credentials_source == 'static'
+        if options['aws']['access_key_id'].nil? || options['aws']['secret_access_key'].nil?
+            raise ArgumentError, "Must use access_key_id and secret_access_key with static credentials_source"
+        end
+      end
+
+      if credentials_source == 'env_or_profile'
+        if !options['aws']['access_key_id'].nil? || !options['aws']['secret_access_key'].nil?
+            raise ArgumentError, "Can't use access_key_id and secret_access_key with env_or_profile credentials_source"
+        end
+      end
+    end
+
     # Generates initial agent settings. These settings will be read by agent
     # from AWS registry (also a BOSH component) on a target instance. Disk
     # conventions for amazon are:
@@ -722,8 +750,11 @@ module Bosh::AwsCloud
     # @param [Hash] network_spec Agent network spec
     # @param [Hash] environment
     # @param [String] root_device_name root device, e.g. /dev/sda1
+    # @param [Hash] block_device_agent_info disk attachment information to merge into the disks section.
+    #   keys are device type ("ephemeral", "raw_ephemeral") and values are array of strings representing the
+    #   path to the block device. It is expected that "ephemeral" has exactly one value.
     # @return [Hash]
-    def initial_agent_settings(agent_id, network_spec, environment, root_device_name)
+    def initial_agent_settings(agent_id, network_spec, environment, root_device_name, block_device_agent_info)
       settings = {
           "vm" => {
               "name" => "vm-#{SecureRandom.uuid}"
@@ -732,11 +763,13 @@ module Bosh::AwsCloud
           "networks" => agent_network_spec(network_spec),
           "disks" => {
               "system" => root_device_name,
-              "ephemeral" => "/dev/sdb",
               "persistent" => {}
           }
       }
 
+      settings["disks"].merge!(block_device_agent_info)
+      settings["disks"]["ephemeral"] = settings["disks"]["ephemeral"][0]["path"]
+
       settings["env"] = environment if environment
       settings.merge(agent_properties)
     end

data/lib/cloud/aws/helpers.rb

@@ -4,7 +4,12 @@ module Bosh::AwsCloud
 
   module Helpers
     def default_ephemeral_disk_mapping
-       { '/dev/sdb' => 'ephemeral0' }
+      [
+        {
+          device_name: '/dev/sdb',
+          virtual_name: 'ephemeral0',
+        }
+      ]
     end
 
     def ebs_ephemeral_disk_mapping(volume_size_in_gb, volume_type)

data/lib/cloud/aws/instance_manager.rb

@@ -5,60 +5,68 @@ module Bosh::AwsCloud
   class InstanceManager
     include Helpers
 
+    class DiskInfo
+      attr_reader :size, :count
+      def initialize(size, count)
+        @size = size
+        @count = count
+      end
+    end
+
     InstanceStorageMap = {
         # previous generation
-        'm1.small' => 160,
-        'm1.medium' => 410,
-        'm1.large' => 420,
-        'm1.xlarge' => 420,
+        'm1.small' => DiskInfo.new(160, 1),
+        'm1.medium' => DiskInfo.new(410, 1),
+        'm1.large' => DiskInfo.new(420, 2),
+        'm1.xlarge' => DiskInfo.new(420, 4),
 
-        'c1.medium' => 350,
-        'c1.xlarge' => 420,
+        'c1.medium' => DiskInfo.new(350, 1),
+        'c1.xlarge' => DiskInfo.new(420, 4),
 
-        'cc2.8xlarge' => 840,
+        'cc2.8xlarge' => DiskInfo.new(840, 4),
 
-        'cg1.4xlarge' => 840,
+        'cg1.4xlarge' => DiskInfo.new(840, 2),
 
-        'm2.xlarge' => 420,
-        'm2.2xlarge' => 850,
-        'm2.4xlarge' => 840,
+        'm2.xlarge' => DiskInfo.new(420, 1),
+        'm2.2xlarge' => DiskInfo.new(850, 1),
+        'm2.4xlarge' => DiskInfo.new(840, 2),
 
-        'cr1.8xlarge' => 120,
+        'cr1.8xlarge' => DiskInfo.new(120, 2),
 
-        'hi1.4xlarge' => 1024,
+        'hi1.4xlarge' => DiskInfo.new(1024, 2),
 
-        'hs1.8xlarge' => 2000,
+        'hs1.8xlarge' => DiskInfo.new(2000, 24),
 
         # current generation
-        'm3.medium' => 4,
-        'm3.large' => 32,
-        'm3.xlarge' => 40,
-        'm3.2xlarge' => 80,
-
-        'c3.large' => 16,
-        'c3.xlarge' => 40,
-        'c3.2xlarge' => 80,
-        'c3.4xlarge' => 160,
-        'c3.8xlarge' => 320,
-
-        'r3.large' => 32,
-        'r3.xlarge' => 80,
-        'r3.2xlarge' => 160,
-        'r3.4xlarge' => 320,
-        'r3.8xlarge' => 320,
-
-        'g2.2xlarge' => 60,
-        'g2.8xlarge' => 120,
-
-        'i2.xlarge' => 800,
-        'i2.2xlarge' => 800,
-        'i2.4xlarge' => 800,
-        'i2.8xlarge' => 800,
-
-        'd2.xlarge' => 2000,
-        'd2.2xlarge' => 2000,
-        'd2.4xlarge' => 2000,
-        'd2.8xlarge' => 2000
+        'm3.medium' => DiskInfo.new(4, 1),
+        'm3.large' => DiskInfo.new(32, 1),
+        'm3.xlarge' => DiskInfo.new(40, 2),
+        'm3.2xlarge' => DiskInfo.new(80, 2),
+
+        'c3.large' => DiskInfo.new(16, 2),
+        'c3.xlarge' => DiskInfo.new(40, 2),
+        'c3.2xlarge' => DiskInfo.new(80, 2),
+        'c3.4xlarge' => DiskInfo.new(160, 2),
+        'c3.8xlarge' => DiskInfo.new(320, 2),
+
+        'r3.large' => DiskInfo.new(32, 1),
+        'r3.xlarge' => DiskInfo.new(80, 1),
+        'r3.2xlarge' => DiskInfo.new(160, 1),
+        'r3.4xlarge' => DiskInfo.new(320, 1),
+        'r3.8xlarge' => DiskInfo.new(320, 2),
+
+        'g2.2xlarge' => DiskInfo.new(60, 1),
+        'g2.8xlarge' => DiskInfo.new(120, 2),
+
+        'i2.xlarge' => DiskInfo.new(800, 1),
+        'i2.2xlarge' => DiskInfo.new(800, 2),
+        'i2.4xlarge' => DiskInfo.new(800, 4),
+        'i2.8xlarge' => DiskInfo.new(800, 8),
+
+        'd2.xlarge' => DiskInfo.new(2000, 3),
+        'd2.2xlarge' => DiskInfo.new(2000, 6),
+        'd2.4xlarge' => DiskInfo.new(2000, 12),
+        'd2.8xlarge' => DiskInfo.new(2000, 24)
     }
 
     def initialize(region, registry, elb, az_selector, logger)
@@ -70,7 +78,7 @@ module Bosh::AwsCloud
     end
 
     def create(agent_id, stemcell_id, resource_pool, networks_spec, disk_locality, environment, options)
-      instance_params = build_instance_params(stemcell_id, resource_pool, networks_spec, disk_locality, options)
+      instance_params, block_device_info = build_instance_params(stemcell_id, resource_pool, networks_spec, disk_locality, options)
 
       @logger.info("Creating new instance with: #{instance_params.inspect}")
 
@@ -93,7 +101,12 @@ module Bosh::AwsCloud
         raise
       end
 
-      instance
+      block_device_agent_info = block_device_info
+                                    .group_by { |v| v[:bosh_type] }
+                                    .map { |type, devices| {type => devices.map { |device| {"path" => device[:device_name] }}}}
+                                    .inject({}) { |a, b| a.merge(b) }
+
+      return instance, block_device_agent_info
     end
 
     # @param [String] instance_id EC2 instance id
@@ -104,13 +117,13 @@ module Bosh::AwsCloud
     private
 
     def build_instance_params(stemcell_id, resource_pool, networks_spec, disk_locality, options)
+      virtualization_type = @region.images[stemcell_id].virtualization_type
+      block_device_info = block_device_mapping(virtualization_type, resource_pool)
+
       instance_params = {count: 1}
       instance_params[:image_id] = stemcell_id
       instance_params[:instance_type] = resource_pool["instance_type"]
-
-      ephemeral_disk_options = resource_pool.fetch("ephemeral_disk", {})
-      ephemeral_disk_options["instance_type"] = resource_pool["instance_type"]
-      instance_params[:block_device_mappings] = block_device_mapping(ephemeral_disk_options)
+      instance_params[:block_device_mappings] = block_device_info.map { |entry| entry.reject{ |k| k == :bosh_type } }
       instance_params[:placement_group] = resource_pool["placement_group"] if resource_pool["placement_group"]
       instance_params[:dedicated_tenancy] = true if resource_pool["tenancy"] == "dedicated"
 
@@ -118,14 +131,15 @@ module Bosh::AwsCloud
       set_key_name_parameter(instance_params, resource_pool["key_name"], options["aws"]["default_key_name"])
       set_security_groups_parameter(instance_params, networks_spec, options["aws"]["default_security_groups"])
       set_vpc_parameters(instance_params, networks_spec)
-
+      set_iam_instance_profile_parameter(instance_params, resource_pool["iam_instance_profile"], options["aws"]["default_iam_instance_profile"])
       set_availability_zone_parameter(
         instance_params,
         (disk_locality || []).map { |volume_id| @region.volumes[volume_id].availability_zone.to_s },
         resource_pool["availability_zone"],
         (instance_params[:subnet].availability_zone_name if instance_params[:subnet])
       )
-      instance_params
+
+      return instance_params, block_device_info
     end
 
     def create_aws_instance(instance_params, spot_bid_price)
@@ -149,15 +163,21 @@ module Bosh::AwsCloud
 
     def instance_create_wait_time; 30; end
 
-    def block_device_mapping(ephemeral_disk_options)
+    def block_device_mapping(virtualization_type, resource_pool)
+      ephemeral_disk_options = resource_pool.fetch("ephemeral_disk", {})
+
       ephemeral_volume_size_in_mb = ephemeral_disk_options.fetch('size', 0)
       ephemeral_volume_size_in_gb = (ephemeral_volume_size_in_mb / 1024.0).ceil
       ephemeral_volume_type = ephemeral_disk_options.fetch('type', 'standard')
-      instance_type = ephemeral_disk_options.fetch('instance_type', '')
+      instance_type = resource_pool.fetch('instance_type', '')
+      raw_instance_storage = resource_pool.fetch('raw_instance_storage', false)
 
-      instance_storage_size_gb = InstanceManager::InstanceStorageMap[instance_type]
+      local_disk_info = InstanceManager::InstanceStorageMap[instance_type]
+      if raw_instance_storage && local_disk_info.nil?
+        raise Bosh::Clouds::CloudError, "raw_instance_storage requested for instance type #{instance_type} that does not have instance storage"
+      end
 
-      if instance_storage_size_gb.nil? || instance_storage_size_gb < ephemeral_volume_size_in_gb
+      if raw_instance_storage || local_disk_info.nil? || local_disk_info.size < ephemeral_volume_size_in_gb
         @logger.debug('Use EBS storage to create the virtual machine')
 
         ephemeral_volume_size_in_gb = 10 if ephemeral_volume_size_in_gb == 0
@@ -167,9 +187,34 @@ module Bosh::AwsCloud
         block_device_mapping_param = default_ephemeral_disk_mapping
       end
 
+      block_device_mapping_param[0][:bosh_type] = 'ephemeral'
+
+      if raw_instance_storage
+        next_device = first_raw_ephemeral_device(virtualization_type)
+        for i in 0..local_disk_info.count - 1 do
+          block_device_mapping_param << {
+              virtual_name: "ephemeral#{i}",
+              device_name: next_device,
+              bosh_type: "raw_ephemeral",
+          }
+          next_device = next_device.next
+        end
+      end
+
       block_device_mapping_param
     end
 
+    def first_raw_ephemeral_device(virtualization_type)
+      case virtualization_type
+        when :hvm
+          '/dev/xvdba'
+        when :paravirtual
+          '/dev/sdc'
+        else
+          raise Bosh::Clouds::CloudError, "unknown virtualization type #{virtualization_type}"
+      end
+    end
+
     def set_key_name_parameter(instance_params, resource_pool_key_name, default_aws_key_name)
       key_name = resource_pool_key_name || default_aws_key_name
       instance_params[:key_name] = key_name unless key_name.nil?
@@ -213,6 +258,11 @@ module Bosh::AwsCloud
       instance_params[:user_data] = Yajl::Encoder.encode(user_data)
     end
 
+    def set_iam_instance_profile_parameter(instance_params, resource_pool_iam_instance_profile, default_aws_iam_instance_profile)
+      iam_instance_profile = resource_pool_iam_instance_profile || default_aws_iam_instance_profile
+      instance_params[:iam_instance_profile] = iam_instance_profile unless iam_instance_profile.nil?
+    end
+
     def validate_and_prepare_security_groups_parameter(instance_params, security_groups)
       return if security_groups.nil? || security_groups.empty?
 

data/lib/cloud/aws/stemcell_creator.rb

@@ -120,10 +120,8 @@ module Bosh::AwsCloud
       params.merge!(
         :name => "BOSH-#{SecureRandom.uuid}",
         :architecture => architecture,
-        :block_device_mappings => params[:block_device_mappings].merge(
-          default_ephemeral_disk_mapping
-        )
       )
+      merge_default_ephemeral_mapping_for_create_disk(params)
 
       params
     end
@@ -131,5 +129,12 @@ module Bosh::AwsCloud
     def logger
       Bosh::Clouds::Config.logger
     end
+
+    # adapts the format of the default ephemeral mapping used by create_vm so it
+    # works with the create AMI, and inserts it into the given hash
+    def merge_default_ephemeral_mapping_for_create_disk(params)
+      ephemeral_mapping = default_ephemeral_disk_mapping[0]
+      params[:block_device_mappings][ephemeral_mapping[:device_name]] = ephemeral_mapping[:virtual_name]
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh_aws_cpi
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-12 00:00:00.000000000 Z
+date: 2015-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -147,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.2.5
 signing_key: 
 specification_version: 4
 summary: BOSH AWS CPI