bosh_aws_cpi 0.6.2 → 0.7.0

data/README.md

@@ -0,0 +1,104 @@
+# BOSH AWS Cloud Provider Interface
+Copyright (c) 2009-2012 VMware, Inc.
+
+For online documentation see: http://rubydoc.info/gems/bosh_aws_cpi/
+
+## Options
+
+These options are passed to the AWS CPI when it is instantiated.
+
+### AWS options
+
+* `access_key_id` (required)  
+  AWS IAM user access key
+* `secret_access_key` (required)  
+  AWS IAM secret access key
+* `default_key_name` (required)  
+  default AWS ssh key name to assign to created virtual machines
+* `default_security_group` (required)  
+  default AWS security group to assign to created virtual machines
+* `ec2_private_key` (required)  
+  local path to the ssh private key, must match `default_key_name`
+* `region` (optional)  
+  EC2 region, defaults to `us-east-1`
+* `ec2_endpoint` (optional)  
+  URL of the EC2 endpoint to connect to, defaults to the endpoint corresponding to the selected region,
+  or `DEFAULT_EC2_ENDPOINT` if no region has been selected
+* `max_retries` (optional)  
+  maximum number of time to retry an AWS API call, defaults to `DEFAULT_MAX_RETRIES`
+
+### Registry options
+
+The registry options are passed to the AWS CPI by the BOSH director based on the settings in `director.yml`, but can be
+overridden if needed.
+
+* `endpoint` (required)  
+  registry URL
+* `user` (required)  
+  registry user
+* `password` (required)  
+  registry password
+
+### Agent options
+
+Agent options are passed to the AWS CPI by the BOSH director based on the settings in `director.yml`, but can be
+overridden if needed.
+
+### Resource pool options
+
+These options are specified under `cloud_options` in the `resource_pools` section of a BOSH deployment manifest.
+
+* `availability_zone` (optional)  
+  the EC2 availability zone the VMs should be created in
+* `instance_type` (required)  
+  which [type of instance](http://aws.amazon.com/ec2/instance-types/) the VMs should belong to
+
+### Network options
+
+These options are specified under `cloud_options` in the `networks` section of a BOSH deployment manifest.
+
+* `type` (required)  
+  can be either `dynamic` for a DHCP assigned IP by AWS, or `vip` to use an Elastic IP (which needs to be already
+  allocated)
+
+## Example
+
+This is a sample of how AWS specific properties are used in a BOSH deployment manifest:
+
+    ---
+    name: sample
+    director_uuid: 38ce80c3-e9e9-4aac-ba61-97c676631b91
+
+    ...
+
+    networks:
+      - name: nginx_network
+        type: vip
+        cloud_properties: {}
+      - name: default
+        type: dynamic
+        cloud_properties:
+          security_groups:
+          - default
+
+    ...
+
+    resource_pools:
+      - name: common
+        network: default
+        size: 3
+        stemcell:
+          name: bosh-stemcell
+          version: 0.6.7
+        cloud_properties:
+          instance_type: m1.small
+
+    ...
+
+    properties:
+      aws:
+        access_key_id: AKIAIYJWVDUP4KRWBESQ
+        secret_access_key: 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
+        default_key_name: bosh
+        default_security_groups: ["bosh"]
+        ec2_private_key: /home/bosh/.ssh/bosh

data/lib/cloud/aws/aki_picker.rb

@@ -0,0 +1,68 @@
+module Bosh::AwsCloud
+  class AKIPicker
+
+    # @param [AWS::Core::ServiceInterface] ec2
+    def initialize(ec2)
+      @ec2 = ec2
+    end
+
+    # finds the correct aki for the current region
+    # @param [String] architecture instruction architecture to find
+    # @param [String] root_device_name
+    # @return [String] EC2 image id
+    def pick(architecture, root_device_name)
+      candidate = pick_candidate(fetch_akis(architecture), root_device_name)
+      raise Bosh::Clouds::CloudError, "unable to find AKI" unless candidate
+      logger.info("auto-selected AKI: #{candidate.image_id}")
+
+      candidate.image_id
+    end
+
+    private
+
+    def fetch_akis(architecture)
+      filter = aki_filter(architecture)
+      @ec2.client.describe_images(:filters => filter).images_set
+    end
+
+    # @return [Hash] search filter
+    def aki_filter(architecture)
+      [
+          {:name => "architecture", :values => [architecture]},
+          {:name => "image-type", :values => %w[kernel]},
+          {:name => "owner-alias", :values => %w[amazon]}
+      ]
+    end
+
+    def regexp(root_device_name)
+      # do nasty hackery to select boot device and version from
+      # the image_location string e.g. pv-grub-hd00_1.03-x86_64.gz
+      if root_device_name == "/dev/sda1"
+        /-hd00[-_](\d+)\.(\d+)/
+      else
+        /-hd0[-_](\d+)\.(\d+)/
+      end
+    end
+
+    # @param [AWS::EC2::ImageCollection] akis
+    def pick_candidate(akis, root_device_name)
+      candidate = nil
+      major = 0
+      minor = 0
+      akis.each do |image|
+        match = image.image_location.match(regexp(root_device_name))
+        if match && match[1].to_i > major && match[2].to_i > minor
+          candidate = image
+          major = match[1].to_i
+          minor = match[2].to_i
+        end
+      end
+
+      candidate
+    end
+
+    def logger
+      Bosh::Clouds::Config.logger
+    end
+  end
+end

data/lib/cloud/aws/cloud.rb

@@ -5,20 +5,26 @@ module Bosh::AwsCloud
   class Cloud < Bosh::Cloud
     include Helpers
 
+    # default maximum number of times to retry an AWS API call
     DEFAULT_MAX_RETRIES = 2
+    # default availability zone for instances and disks
     DEFAULT_AVAILABILITY_ZONE = "us-east-1a"
     DEFAULT_EC2_ENDPOINT = "ec2.amazonaws.com"
-    METADATA_TIMEOUT = 5 # seconds
-    DEVICE_POLL_TIMEOUT = 60 # seconds
+    METADATA_TIMEOUT = 5 # in seconds
+    DEVICE_POLL_TIMEOUT = 60 # in seconds
+    MAX_TAG_KEY_LENGTH = 127
+    MAX_TAG_VALUE_LENGTH = 255
 
     attr_reader :ec2
     attr_reader :registry
     attr_accessor :logger
 
     ##
-    # Initialize BOSH AWS CPI
+    # Initialize BOSH AWS CPI. The contents of sub-hashes are defined in the {file:README.md}
     # @param [Hash] options CPI options
-    #
+    # @option options [Hash] aws AWS specific options
+    # @option options [Hash] agent agent options
+    # @option options [Hash] registry agent options
     def initialize(options)
       @options = options.dup
 
@@ -30,6 +36,7 @@ module Bosh::AwsCloud
 
       @agent_properties = @options["agent"] || {}
       @aws_properties = @options["aws"]
+      @aws_region = @aws_properties.delete("region")
       @registry_properties = @options["registry"]
 
       @default_key_name = @aws_properties["default_key_name"]
@@ -38,7 +45,7 @@ module Bosh::AwsCloud
       aws_params = {
         :access_key_id => @aws_properties["access_key_id"],
         :secret_access_key => @aws_properties["secret_access_key"],
-        :ec2_endpoint => @aws_properties["ec2_endpoint"] || DEFAULT_EC2_ENDPOINT,
+        :ec2_endpoint => @aws_properties["ec2_endpoint"] || default_ec2_endpoint,
         :max_retries => @aws_properties["max_retries"] || DEFAULT_MAX_RETRIES,
         :logger => @aws_logger
       }
@@ -59,37 +66,31 @@ module Bosh::AwsCloud
                                      registry_user,
                                      registry_password)
 
+      @aki_picker = AKIPicker.new(@ec2)
       @metadata_lock = Mutex.new
     end
 
     ##
-    # Creates EC2 instance and waits until it's in running state
-    # @param [String] agent_id Agent id associated with new VM
-    # @param [String] stemcell_id AMI id that will be used
-    #   to power on new instance
-    # @param [Hash] resource_pool Resource pool specification
-    # @param [Hash] network_spec Network specification, if it contains
-    #  security groups they must be existing
-    # @param [optional, Array] disk_locality List of disks that
+    # Create an EC2 instance and wait until it's in running state
+    # @param [String] agent_id agent id associated with new VM
+    # @param [String] stemcell_id AMI id of the stemcell used to
+    #  create the new instance
+    # @param [Hash] resource_pool resource pool specification
+    # @param [Hash] network_spec network specification, if it contains
+    #  security groups they must already exist
+    # @param [optional, Array] disk_locality list of disks that
     #   might be attached to this instance in the future, can be
     #   used as a placement hint (i.e. instance will only be created
     #   if resource pool availability zone is the same as disk
     #   availability zone)
-    # @param [optional, Hash] environment Data to be merged into
+    # @param [optional, Hash] environment data to be merged into
     #   agent settings
-    #
-    # @return [String] created instance id
+    # @return [String] EC2 instance id of the new virtual machine
     def create_vm(agent_id, stemcell_id, resource_pool,
                   network_spec, disk_locality = nil, environment = nil)
       with_thread_name("create_vm(#{agent_id}, ...)") do
         network_configurator = NetworkConfigurator.new(network_spec)
 
-        user_data = {
-          "registry" => {
-            "endpoint" => @registry.endpoint
-          }
-        }
-
         security_groups =
           network_configurator.security_groups(@default_security_groups)
         @logger.debug("using security groups: #{security_groups.join(', ')}")
@@ -107,7 +108,7 @@ module Bosh::AwsCloud
           :key_name => resource_pool["key_name"] || @default_key_name,
           :security_groups => security_groups,
           :instance_type => resource_pool["instance_type"],
-          :user_data => Yajl::Encoder.encode(user_data)
+          :user_data => Yajl::Encoder.encode(user_data(network_spec))
         }
 
         instance_params[:availability_zone] =
@@ -131,8 +132,9 @@ module Bosh::AwsCloud
     end
 
     ##
-    # Terminates EC2 instance and waits until it reports as terminated
-    # @param [String] instance_id Running instance id
+    # Delete EC2 instance ("terminate" in AWS language) and wait until
+    # it reports as terminated
+    # @param [String] instance_id EC2 instance id
     def delete_vm(instance_id)
       with_thread_name("delete_vm(#{instance_id})") do
         instance = @ec2.instances[instance_id]
@@ -153,8 +155,8 @@ module Bosh::AwsCloud
     end
 
     ##
-    # Reboots EC2 instance
-    # @param [String] instance_id Running instance id
+    # Reboot EC2 instance
+    # @param [String] instance_id EC2 instance id
     def reboot_vm(instance_id)
       with_thread_name("reboot_vm(#{instance_id})") do
         instance = @ec2.instances[instance_id]
@@ -165,7 +167,7 @@ module Bosh::AwsCloud
     ##
     # Creates a new EBS volume
     # @param [Integer] size disk size in MiB
-    # @param [optional, String] instance_id vm id
+    # @param [optional, String] instance_id EC2 instance id
     #        of the VM that this disk will be attached to
     # @return [String] created EBS volume id
     def create_disk(size, instance_id = nil)
@@ -182,11 +184,13 @@ module Bosh::AwsCloud
           cloud_error("AWS CPI maximum disk size is 1 TiB")
         end
 
+        # if the disk is created for an instance, use the same availability
+        # zone as they must match
         if instance_id
           instance = @ec2.instances[instance_id]
           availability_zone = instance.availability_zone
         else
-          availability_zone = DEFAULT_AVAILABILITY_ZONE
+          availability_zone = default_availability_zone
         end
 
         volume_params = {
@@ -203,10 +207,9 @@ module Bosh::AwsCloud
     end
 
     ##
-    # Deletes EBS volume
-    # @param [String] disk_id volume id
+    # Delete EBS volume
+    # @param [String] disk_id EBS volume id
     # @raise [Bosh::Clouds::CloudError] if disk is not in available state
-    # @return nil
     def delete_disk(disk_id)
       with_thread_name("delete_disk(#{disk_id})") do
         volume = @ec2.volumes[disk_id]
@@ -229,6 +232,9 @@ module Bosh::AwsCloud
       end
     end
 
+    # Attach an EBS volume to an EC2 instance
+    # @param [String] instance_id EC2 instance id of the virtual machine to attach the disk to
+    # @param [String] disk_id EBS volume id of the disk to attach
     def attach_disk(instance_id, disk_id)
       with_thread_name("attach_disk(#{instance_id}, #{disk_id})") do
         instance = @ec2.instances[instance_id]
@@ -241,9 +247,13 @@ module Bosh::AwsCloud
           settings["disks"]["persistent"] ||= {}
           settings["disks"]["persistent"][disk_id] = device_name
         end
+        @logger.info("Attached `#{disk_id}' to `#{instance_id}'")
       end
     end
 
+    # Detach an EBS volume from an EC2 instance
+    # @param [String] instance_id EC2 instance id of the virtual machine to detach the disk from
+    # @param [String] disk_id EBS volume id of the disk to detach
     def detach_disk(instance_id, disk_id)
       with_thread_name("detach_disk(#{instance_id}, #{disk_id})") do
         instance = @ec2.instances[instance_id]
@@ -261,10 +271,10 @@ module Bosh::AwsCloud
       end
     end
 
-    # Configures network for a running instance
-    # @param [String] instance_id instance identifier
+    # Configure network for an EC2 instance
+    # @param [String] instance_id EC2 instance id
     # @param [Hash] network_spec network properties
-    # @raises [Bosh::Clouds:NotSupported] if the security groups change
+    # @raise [Bosh::Clouds:NotSupported] if the security groups change
     def configure_networks(instance_id, network_spec)
       with_thread_name("configure_networks(#{instance_id}, ...)") do
         @logger.info("Configuring `#{instance_id}' to use the following " \
@@ -294,24 +304,26 @@ module Bosh::AwsCloud
     end
 
     ##
-    # Creates a new AMI using stemcell image.
+    # Creates a new EC2 AMI using stemcell image.
     # This method can only be run on an EC2 instance, as image creation
     # involves creating and mounting new EBS volume as local block device.
     # @param [String] image_path local filesystem path to a stemcell image
-    # @param [Hash] cloud_properties CPI-specific properties
+    # @param [Hash] cloud_properties AWS-specific stemcell properties
     # @option cloud_properties [String] kernel_id
-    #   AKI, auto-selected based on the region
+    #   AKI, auto-selected based on the region, unless specified
     # @option cloud_properties [String] root_device_name
-    #   provided by the stemcell manifest
+    #   block device path (e.g. /dev/sda1), provided by the stemcell manifest, unless specified
     # @option cloud_properties [String] architecture
-    #   provided by the stemcell manifest
+    #   instruction set architecture (e.g. x86_64), provided by the stemcell manifest,
+    #   unless specified
     # @option cloud_properties [String] disk (2048)
     #   root disk size
+    # @return [String] EC2 AMI name of the stemcell
     def create_stemcell(image_path, cloud_properties)
       # TODO: refactor into several smaller methods
       with_thread_name("create_stemcell(#{image_path}...)") do
         begin
-          # These two variables are used in 'ensure' clause
+          # These three variables are used in 'ensure' clause
           instance = nil
           volume = nil
           # 1. Create and mount new EBS volume (2GB default)
@@ -331,31 +343,14 @@ module Bosh::AwsCloud
           snapshot = volume.create_snapshot
           wait_resource(snapshot, :completed)
 
-          root_device_name = cloud_properties["root_device_name"]
-          architecture = cloud_properties["architecture"]
-
-          aki = find_aki(architecture, root_device_name)
-
-          # we could set :description here, but since we don't have a
-          # handle to the stemcell name and version, we can't set it
-          # to something useful :(
-          image_params = {
-            :name => "BOSH-#{generate_unique_name}",
-            :architecture => architecture,
-            :kernel_id => aki,
-            :root_device_name =>  root_device_name,
-            :block_device_mappings => {
-              "/dev/sda" => { :snapshot_id => snapshot.id },
-              "/dev/sdb" => "ephemeral0"
-            }
-          }
-
-          image = @ec2.images.create(image_params)
+          params = image_params(cloud_properties, snapshot.id)
+          image = @ec2.images.create(params)
           wait_resource(image, :available, :state)
 
+          tag(image, "Name", params[:description]) if params[:description]
+
           image.id
         rescue => e
-          # TODO: delete snapshot?
           @logger.error(e)
           raise e
         ensure
@@ -367,49 +362,53 @@ module Bosh::AwsCloud
       end
     end
 
-    # finds the correct aki for the current region
-    def find_aki(arch, root_device_name)
-
-      filters = []
-      filters << {:name => "architecture", :values => [arch]}
-      filters << {:name => "image-type", :values => %w[kernel]}
-      filters << {:name => "owner-alias", :values => %w[amazon]}
+    # Delete a stemcell and the accompanying snapshots
+    # @param [String] stemcell_id EC2 AMI name of the stemcell to be deleted
+    def delete_stemcell(stemcell_id)
+      with_thread_name("delete_stemcell(#{stemcell_id})") do
+        snapshots = []
+        image = @ec2.images[stemcell_id]
 
-      response = @ec2.client.describe_images(:filters => filters)
+        image.block_device_mappings.each do |device, map|
+          id = map[:snapshot_id]
+          if id
+            @logger.debug("queuing snapshot #{id} for deletion")
+            snapshots << id
+          end
+        end
 
-      # do nasty hackery to select boot device and version from
-      # the image_location string e.g. pv-grub-hd00_1.03-x86_64.gz
-      if root_device_name == "/dev/sda1"
-        regexp = /-hd00[-_](\d+)\.(\d+)/
-      else
-        regexp = /-hd0[-_](\d+)\.(\d+)/
-      end
+        image.deregister
 
-      candidate = nil
-      major = 0
-      minor = 0
-      response.images_set.each do |image|
-        match = image.image_location.match(regexp)
-        if match && match[1].to_i > major && match[2].to_i > minor
-          candidate = image
-          major = match[1].to_i
-          minor = match[2].to_i
+        snapshots.each do |id|
+          @logger.info("cleaning up snapshot #{id}")
+          snapshot = @ec2.snapshots[id]
+          snapshot.delete
         end
       end
-
-      cloud_error("unable to find AKI") unless candidate
-      @logger.info("auto-selected AKI: #{candidate.image_id}")
-
-      candidate.image_id
     end
 
-    def delete_stemcell(stemcell_id)
-      with_thread_name("delete_stemcell(#{stemcell_id})") do
-        image = @ec2.images[stemcell_id]
-        image.deregister
+    # Add tags to an instance. In addition to the suplied tags,
+    # it adds a 'Name' tag as it is shown in the AWS console.
+    # @param [String] vm vm id that was once returned by {#create_vm}
+    # @param [Hash] metadata metadata key/value pairs
+    # @return [void]
+    def set_vm_metadata(vm, metadata)
+      instance = @ec2.instances[vm]
+
+      # TODO should we clear existing tags that don't exist in metadata?
+      metadata.each_pair do |key, value|
+        tag(instance, key, value)
       end
+
+      # should deployment name be included too?
+      job = metadata[:job]
+      index = metadata[:index]
+      tag(instance, "Name", "#{job}/#{index}") if job && index
+    rescue AWS::EC2::Errors::TagLimitExceeded => e
+      @logger.error("could not tag #{instance.id}: #{e.message}")
     end
 
+    # @note Not implemented in the AWS CPI
     def validate_deployment(old_manifest, new_manifest)
       # Not implemented in VSphere CPI as well
       not_implemented(:validate_deployment)
@@ -422,17 +421,19 @@ module Bosh::AwsCloud
     # @param [String] resource_pool_az availability zone specified in
     #   the resource pool (may be nil)
     # @return [String] availability zone to use
+    # @note this is a private method that is public to make it easier to test
     def select_availability_zone(volumes, resource_pool_az)
       if volumes && !volumes.empty?
         disks = volumes.map { |vid| @ec2.volumes[vid] }
         ensure_same_availability_zone(disks, resource_pool_az)
         disks.first.availability_zone
       else
-        resource_pool_az || DEFAULT_AVAILABILITY_ZONE
+        resource_pool_az || default_availability_zone
       end
     end
 
     # ensure all supplied availability zones are the same
+    # @note this is a private method that is public to make it easier to test
     def ensure_same_availability_zone(disks, default)
       zones = disks.map { |disk| disk.availability_zone }
       zones << default if default
@@ -443,6 +444,70 @@ module Bosh::AwsCloud
 
     private
 
+    # add a tag to something
+    def tag(taggable, key, value)
+      trimmed_key = key[0..(MAX_TAG_KEY_LENGTH - 1)]
+      trimmed_value = value[0..(MAX_TAG_VALUE_LENGTH - 1)]
+      taggable.add_tag(trimmed_key, :value => trimmed_value)
+    rescue AWS::EC2::Errors::InvalidParameterValue => e
+      @logger.error("could not tag #{taggable.id}: #{e.message}")
+    end
+
+    # Prepare EC2 user data
+    # @param [Hash] network_spec network specification
+    # @return [Hash] EC2 user data
+    def user_data(network_spec)
+      data = {}
+
+      data["registry"] = { "endpoint" => @registry.endpoint }
+
+      with_dns(network_spec) do |servers|
+        data["dns"] = { "nameserver" => servers }
+      end
+
+      data
+    end
+
+    # extract dns server list from network spec and yield the the list
+    # @param [Hash] network_spec network specification for instance
+    # @yield [Array]
+    def with_dns(network_spec)
+      network_spec.each_value do |properties|
+        if properties["dns"]
+          yield properties["dns"]
+          return
+        end
+      end
+    end
+
+    def image_params(cloud_properties, snapshot_id)
+      root_device_name = cloud_properties["root_device_name"]
+      architecture = cloud_properties["architecture"]
+
+      params = {
+          :name => "BOSH-#{generate_unique_name}",
+          :architecture => architecture,
+          :kernel_id => find_aki(architecture, root_device_name),
+          :root_device_name =>  root_device_name,
+          :block_device_mappings => {
+              "/dev/sda" => { :snapshot_id => snapshot_id },
+              "/dev/sdb" => "ephemeral0"
+          }
+      }
+
+      # old stemcells doesn't have name & version
+      if cloud_properties["name"] && cloud_properties["version"]
+        name = "#{cloud_properties['name']} #{cloud_properties['version']}"
+        params[:description] = name
+      end
+
+      params
+    end
+
+    def find_aki(architecture, root_device_name)
+      @aki_picker.pick(architecture, root_device_name)
+    end
+
     ##
     # Generates initial agent settings. These settings will be read by agent
     # from AWS registry (also a BOSH component) on a target instance. Disk
@@ -571,10 +636,11 @@ module Bosh::AwsCloud
       attachment = volume.detach_from(instance, device_map[volume.id])
       @logger.info("Detaching `#{volume.id}' from `#{instance.id}'")
 
-      begin
-        wait_resource(attachment, :detached)
-      rescue AWS::Core::Resource::NotFound
-        # It's OK, just means attachment is gone by now
+      wait_resource(attachment, :detached) do |error|
+        if error.is_a? AWS::Core::Resource::NotFound
+          @logger.info("attachment is no longer found, assuming it to be detached")
+          :detached
+        end
       end
     end
 
@@ -678,6 +744,25 @@ module Bosh::AwsCloud
       end
     end
 
+    def default_ec2_endpoint
+      if @aws_region
+        "ec2.#{@aws_region}.amazonaws.com"
+      else
+        DEFAULT_EC2_ENDPOINT
+      end
+    end
+
+    def default_availability_zone
+      if @aws_region
+        "#{@aws_region}b"
+      else
+        DEFAULT_AVAILABILITY_ZONE
+      end
+    end
+
+    def task_checkpoint
+      Bosh::Clouds::Config.task_checkpoint
+    end
   end
 
 end

data/lib/cloud/aws/helpers.rb

@@ -21,9 +21,13 @@ module Bosh::AwsCloud
 
       started_at = Time.now
       failures = 0
-      desc = resource.to_s
+
+      # all resources but Attachment have id
+      desc = resource.respond_to?(:id) ? resource.id : resource.to_s
 
       loop do
+        task_checkpoint
+
         duration = Time.now - started_at
 
         if duration > timeout
@@ -35,21 +39,13 @@ module Bosh::AwsCloud
                         "(#{duration}s)")
         end
 
-        begin
-          state = resource.send(state_method)
-        rescue AWS::EC2::Errors::InvalidAMIID::NotFound,
-               AWS::EC2::Errors::InvalidInstanceID::NotFound => e
-          # ugly workaround for AWS race conditions:
-          # 1) sometimes when we upload a stemcell and proceed to create a VM
-          #    from it, AWS reports that the AMI is missing
-          # 2) sometimes when we create a new EC2 instance, AWS reports that
-          #    the instance it returns is missing
-          # in both cases we just wait a little and retry...
-          raise e if failures > 3
-          failures += 1
-          @logger.error("#{e.message}: #{desc}")
-          sleep(1)
-          next
+        state = get_state_for(resource, state_method) do |error|
+          if block_given?
+            yield error
+          else
+            @logger.error("#{error.message}: #{desc}")
+            nil
+          end
         end
 
         # This is not a very strong convention, but some resources
@@ -71,6 +67,22 @@ module Bosh::AwsCloud
         @logger.info("#{desc} is now #{target_state}, took #{total}s")
       end
     end
+
+    private
+
+    def get_state_for(resource, state_method)
+      resource.send(state_method)
+    rescue AWS::EC2::Errors::InvalidAMIID::NotFound,
+        AWS::EC2::Errors::InvalidInstanceID::NotFound,
+        AWS::Core::Resource::NotFound => e
+      # ugly workaround for AWS race conditions:
+      # 1) sometimes when we upload a stemcell and proceed to create a VM
+      #    from it, AWS reports that the AMI is missing
+      # 2) sometimes when we create a new EC2 instance, AWS reports that
+      #    the instance it returns is missing
+      # in both cases we just catch the exception, wait a little and retry...
+      yield e
+    end
   end
 end
 

data/lib/cloud/aws/registry_client.rb

@@ -42,7 +42,7 @@ module Bosh::AwsCloud
       payload = Yajl::Encoder.encode(settings)
       url = "#{@endpoint}/instances/#{instance_id}/settings"
 
-      response = @client.put(url, payload, @headers)
+      response = @client.put(url, {:body => payload, :header => @headers})
 
       if response.status != 200
         cloud_error("Cannot update settings for `#{instance_id}', " \
@@ -59,7 +59,7 @@ module Bosh::AwsCloud
     def read_settings(instance_id)
       url = "#{@endpoint}/instances/#{instance_id}/settings"
 
-      response = @client.get(url, {}, @headers)
+      response = @client.get(url, {:header => @headers})
 
       if response.status != 200
         cloud_error("Cannot read settings for `#{instance_id}', " \
@@ -94,7 +94,7 @@ module Bosh::AwsCloud
     def delete_settings(instance_id)
       url = "#{@endpoint}/instances/#{instance_id}/settings"
 
-      response = @client.delete(url, @headers)
+      response = @client.delete(url, {:header => @headers})
 
       if response.status != 200
         cloud_error("Cannot delete settings for `#{instance_id}', " \

data/lib/cloud/aws/version.rb

@@ -2,6 +2,6 @@
 
 module Bosh
   module AwsCloud
-    VERSION = "0.6.2"
+    VERSION = "0.7.0"
   end
 end

data/lib/cloud/aws.rb

@@ -21,6 +21,7 @@ require "cloud/aws/cloud"
 require "cloud/aws/registry_client"
 require "cloud/aws/version"
 
+require "cloud/aws/aki_picker"
 require "cloud/aws/network_configurator"
 require "cloud/aws/network"
 require "cloud/aws/dynamic_network"

data/spec/spec_helper.rb

@@ -68,21 +68,23 @@ def mock_registry(endpoint = "http://registry:3333")
 end
 
 def mock_cloud(options = nil)
-  instances = double("instances")
-  volumes = double("volumes")
-  images = double("images")
+  ec2 = mock_ec2
+  AWS::EC2.stub(:new).and_return(ec2)
 
-  ec2 = double(AWS::EC2)
+  yield ec2 if block_given?
 
-  ec2.stub(:instances).and_return(instances)
-  ec2.stub(:volumes).and_return(volumes)
-  ec2.stub(:images).and_return(images)
+  Bosh::AwsCloud::Cloud.new(options || mock_cloud_options)
+end
 
-  AWS::EC2.stub(:new).and_return(ec2)
+def mock_ec2
+  ec2 = double(AWS::EC2,
+               :instances => double("instances"),
+               :volumes => double("volumes"),
+               :images =>double("images"))
 
   yield ec2 if block_given?
 
-  Bosh::AwsCloud::Cloud.new(options || mock_cloud_options)
+  ec2
 end
 
 def dynamic_network_spec

data/spec/unit/aki_picker_spec.rb

@@ -0,0 +1,29 @@
+require "spec_helper"
+
+describe Bosh::AwsCloud::AKIPicker do
+  let(:akis) {
+    [
+      double("image-1", :root_device_name => "/dev/sda1",
+             :image_location => "pv-grub-hd00_1.03-x86_64.gz",
+             :image_id => "aki-b4aa75dd"),
+      double("image-2", :root_device_name => "/dev/sda1",
+             :image_location => "pv-grub-hd00_1.02-x86_64.gz",
+             :image_id => "aki-b4aa75d0")
+    ]
+  }
+  let(:logger) {double("logger", :info => nil)}
+  let(:picker) {Bosh::AwsCloud::AKIPicker.new(double("ec2"))}
+
+  it "should pick the AKI with the highest version" do
+    picker.should_receive(:logger).and_return(logger)
+    picker.should_receive(:fetch_akis).and_return(akis)
+    picker.pick("x86_64", "/dev/sda1").should == "aki-b4aa75dd"
+  end
+
+  it "should raise an error when it can't pick an AKI" do
+    picker.should_receive(:fetch_akis).and_return(akis)
+    expect {
+      picker.pick("foo", "bar")
+    }.to raise_error Bosh::Clouds::CloudError, "unable to find AKI"
+  end
+end

data/spec/unit/create_stemcell_spec.rb

@@ -1,6 +1,6 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
-require File.expand_path("../../spec_helper", __FILE__)
+require "spec_helper"
 
 describe Bosh::AwsCloud::Cloud do
 
@@ -8,6 +8,10 @@ describe Bosh::AwsCloud::Cloud do
     @tmp_dir = Dir.mktmpdir
   end
 
+  after(:each) do
+    FileUtils.rm_rf(@tmp_dir)
+  end
+
   describe "EBS-volume based flow" do
 
     it "creates stemcell by copying an image to a new EBS volume" do
@@ -19,7 +23,7 @@ describe Bosh::AwsCloud::Cloud do
                           :device => "/dev/sdh",
                           :volume => volume)
 
-      snapshot = double("snapshot", :id => "s-baz")
+      snapshot = double("snapshot", :id => "s-baz", :delete => nil)
       image = double("image", :id => "i-bar")
 
       unique_name = UUIDTools::UUID.random_create.to_s
@@ -29,29 +33,25 @@ describe Bosh::AwsCloud::Cloud do
         :architecture => "x86_64",
         :kernel_id => "aki-b4aa75dd",
         :root_device_name => "/dev/sda1",
+        :description => "bosh-stemcell 1.2.3",
         :block_device_mappings => {
           "/dev/sda" => { :snapshot_id => "s-baz" },
           "/dev/sdb" => "ephemeral0"
         }
       }
 
+
+      image.should_receive(:add_tag).with("Name", {:value=>"bosh-stemcell 1.2.3"})
+
       cloud = mock_cloud do |ec2|
         ec2.volumes.stub(:[]).with("v-foo").and_return(volume)
         ec2.instances.stub(:[]).with("i-current").and_return(current_instance)
         ec2.images.should_receive(:create).with(image_params).and_return(image)
-
-        i1 = double("image-1", :root_device_name => "/dev/sda1",
-                       :image_location => "pv-grub-hd00_1.03-x86_64.gz",
-                       :image_id => "aki-b4aa75dd")
-        i2 = double("image-2", :root_device_name => "/dev/sda1",
-                    :image_location => "pv-grub-hd00_1.02-x86_64.gz",
-                    :image_id => "aki-b4aa75d0")
-
-        result = double("images_set", :images_set => [i1, i2])
-        client = double("client", :describe_images => result)
-        ec2.should_receive(:client).and_return(client)
       end
 
+      cloud.should_receive(:find_aki).with("x86_64", "/dev/sda1")
+        .and_return("aki-b4aa75dd")
+
       cloud.stub(:generate_unique_name).and_return(unique_name)
       cloud.stub(:current_instance_id).and_return("i-current")
 
@@ -102,7 +102,9 @@ describe Bosh::AwsCloud::Cloud do
 
       cloud_properties = {
           "root_device_name" => "/dev/sda1",
-          "architecture" => "x86_64"
+          "architecture" => "x86_64",
+          "name" => "bosh-stemcell",
+          "version" => "1.2.3"
       }
       cloud.create_stemcell("/tmp/foo", cloud_properties).should == "i-bar"
     end

data/spec/unit/create_vm_spec.rb

@@ -1,6 +1,6 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
-require File.expand_path("../../spec_helper", __FILE__)
+require "spec_helper"
 
 describe Bosh::AwsCloud::Cloud, "create_vm" do
 
@@ -81,6 +81,46 @@ describe Bosh::AwsCloud::Cloud, "create_vm" do
     vm_id.should == "i-test"
   end
 
+  it "passes dns servers in ec2 user data when present" do
+    unique_name = UUIDTools::UUID.random_create.to_s
+
+    user_data = {
+        "registry" => {
+            "endpoint" => "http://registry:3333"
+        },
+        "dns" => { "nameserver" => ["1.2.3.4"] }
+    }
+
+    sec_grp = double("security_group", :name => "default")
+    instance = double("instance",
+                      :id => "i-test",
+                      :elastic_ip => nil,
+                      :security_groups => sec_grp)
+    client = double("client", :describe_images => fake_image_set)
+
+    network_spec = dynamic_network_spec
+    network_spec["dns"] = ["1.2.3.4"]
+
+    cloud = mock_cloud do |ec2|
+      ec2.instances.should_receive(:create).
+          with(ec2_params(user_data, %w[default])).
+          and_return(instance)
+      ec2.should_receive(:client).and_return(client)
+    end
+
+    cloud.should_receive(:generate_unique_name).and_return(unique_name)
+    cloud.should_receive(:wait_resource).with(instance, :running)
+    @registry.should_receive(:update_settings)
+      .with("i-test", agent_settings(unique_name, network_spec))
+
+    vm_id = cloud.create_vm("agent-id", "sc-id",
+                            resource_pool_spec,
+                            { "network_a" => network_spec },
+                            nil, { "test_env" => "value" })
+
+    vm_id.should == "i-test"
+  end
+
   it "creates EC2 instance with security group" do
     unique_name = UUIDTools::UUID.random_create.to_s
 

data/spec/unit/delete_stemcell_spec.rb

@@ -7,11 +7,22 @@ describe Bosh::AwsCloud::Cloud do
   it "deregisters EC2 image" do
     image = double("image", :id => "i-foo")
 
+    snapshot = double("snapshot")
+    snapshot.should_receive(:delete)
+
+    snapshots = double("snapshots")
+    snapshots.should_receive(:[]).with("snap-123").and_return(snapshot)
+
     cloud = mock_cloud do |ec2|
       ec2.images.stub(:[]).with("i-foo").and_return(image)
+      ec2.should_receive(:snapshots).and_return(snapshots)
     end
 
     image.should_receive(:deregister)
+
+    map = { "/dev/sda" => {:snapshot_id => "snap-123"} }
+    image.should_receive(:block_device_mappings).and_return(map)
+
     cloud.delete_stemcell("i-foo")
   end
 

data/spec/unit/helpers_spec.rb

@@ -4,15 +4,16 @@ require File.expand_path("../../spec_helper", __FILE__)
 
 describe Bosh::AwsCloud::Helpers do
   it "should time out" do
+    Bosh::Clouds::Config.stub(:task_checkpoint)
     cloud = mock_cloud
 
     resource = double("resource")
     resource.stub(:status).and_return(:start)
     cloud.stub(:sleep)
 
-    lambda {
+    expect {
       cloud.wait_resource(resource, :stop, :status, 0.1)
-    }.should raise_error Bosh::Clouds::CloudError, /Timed out/
+    }.to raise_error Bosh::Clouds::CloudError, /Timed out/
   end
 
   it "should not time out" do
@@ -28,15 +29,36 @@ describe Bosh::AwsCloud::Helpers do
   end
 
   it "should raise error when target state is wrong" do
+    Bosh::Clouds::Config.stub(:task_checkpoint)
     cloud = mock_cloud
 
     resource = double("resource")
     resource.stub(:status).and_return(:started, :failed)
     cloud.stub(:sleep)
 
-    lambda {
+    expect {
       cloud.wait_resource(resource, :stopped, :status, 0.1)
-    }.should raise_error Bosh::Clouds::CloudError,
-                         /is failed, expected stopped/
+    }.to raise_error Bosh::Clouds::CloudError, /is failed, expected stopped/
+  end
+
+  it "should swallow AWS::EC2::Errors::InvalidInstanceID::NotFound" do
+    Bosh::Clouds::Config.stub(:task_checkpoint)
+    cloud = mock_cloud
+
+    resource = double("resource")
+    return_values = [:raise, :raise, :raise, :start, :start, :stop]
+    i = 0
+    resource.stub(:status) do
+      i += 1
+      if return_values[i] == :raise
+        raise AWS::EC2::Errors::InvalidInstanceID::NotFound
+      end
+      return_values[i]
+    end
+    cloud.stub(:sleep)
+
+    #lambda {
+      cloud.wait_resource(resource, :stop, :status, 0.1)
+    #}.should_not raise_error AWS::EC2::Errors::InvalidInstanceID::NotFound
   end
 end

data/spec/unit/set_vm_metadata_spec.rb

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "spec_helper"
+
+describe Bosh::AwsCloud::Cloud, "#set_vm_metadata" do
+  before :each do
+    @instance = double("instance", :id => "i-foobar")
+
+    @cloud = mock_cloud(mock_cloud_options) do |ec2|
+      ec2.instances.stub(:[]).with("i-foobar").and_return(@instance)
+    end
+  end
+
+  it "should add new tags" do
+    metadata = {:job => "job", :index => "index"}
+    @cloud.should_receive(:tag).with(@instance, :job, "job")
+    @cloud.should_receive(:tag).with(@instance, :index, "index")
+    @cloud.should_receive(:tag).with(@instance, "Name", "job/index")
+    @cloud.set_vm_metadata("i-foobar", metadata)
+  end
+
+  it "should trim key and value length" do
+    metadata = {"x"*128 => "y"*256}
+    @instance.should_receive(:add_tag) do |key, options|
+      key.size.should == 127
+      options[:value].size.should == 255
+    end
+    @cloud.set_vm_metadata("i-foobar", metadata)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bosh_aws_cpi
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-10 00:00:00.000000000 Z
+date: 2013-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.4.4
+        version: 0.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.4.4
+        version: 0.5.1
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -117,6 +117,7 @@ files:
 - bin/bosh_aws_console
 - lib/bosh_aws_cpi.rb
 - lib/cloud/aws.rb
+- lib/cloud/aws/aki_picker.rb
 - lib/cloud/aws/cloud.rb
 - lib/cloud/aws/dynamic_network.rb
 - lib/cloud/aws/helpers.rb
@@ -125,11 +126,12 @@ files:
 - lib/cloud/aws/registry_client.rb
 - lib/cloud/aws/version.rb
 - lib/cloud/aws/vip_network.rb
-- README
+- README.md
 - Rakefile
 - spec/assets/stemcell-copy
 - spec/integration/cpi_test.rb
 - spec/spec_helper.rb
+- spec/unit/aki_picker_spec.rb
 - spec/unit/attach_disk_spec.rb
 - spec/unit/cloud_spec.rb
 - spec/unit/configure_networks_spec.rb
@@ -143,6 +145,7 @@ files:
 - spec/unit/helpers_spec.rb
 - spec/unit/network_configurator_spec.rb
 - spec/unit/reboot_vm_spec.rb
+- spec/unit/set_vm_metadata_spec.rb
 - spec/unit/validate_deployment_spec.rb
 homepage: http://www.vmware.com
 licenses: []
@@ -158,7 +161,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2760126620918151960
+      hash: -1915781663242535072
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -167,7 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2760126620918151960
+      hash: -1915781663242535072
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -178,6 +181,7 @@ test_files:
 - spec/assets/stemcell-copy
 - spec/integration/cpi_test.rb
 - spec/spec_helper.rb
+- spec/unit/aki_picker_spec.rb
 - spec/unit/attach_disk_spec.rb
 - spec/unit/cloud_spec.rb
 - spec/unit/configure_networks_spec.rb
@@ -191,4 +195,6 @@ test_files:
 - spec/unit/helpers_spec.rb
 - spec/unit/network_configurator_spec.rb
 - spec/unit/reboot_vm_spec.rb
+- spec/unit/set_vm_metadata_spec.rb
 - spec/unit/validate_deployment_spec.rb
+has_rdoc: 

data/README

@@ -1,3 +0,0 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
-BOSH AWS Cloud Provider Interface