bosh_aws_cpi 0.3.3 → 0.5.0

data/bin/bosh_aws_console

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+
+# Copyright (c) 2009-2012 VMware, Inc.
+
+# Usage example:
+# irb(main):001:0> cpi.create_vm("test", "ami-809a48e9",
+#                  {"instance_type" => "m1.small"}, {}, [], {"foo" =>"bar"})
+
+gemfile = File.expand_path("../../Gemfile", __FILE__)
+
+if File.exists?(gemfile)
+  ENV["BUNDLE_GEMFILE"] = gemfile
+  require "rubygems"
+  require "bundler/setup"
+end
+
+$:.unshift(File.expand_path("../../lib", __FILE__))
+require "bosh_aws_cpi"
+require "irb"
+require "irb/completion"
+require "ostruct"
+require "optparse"
+
+config_file = nil
+
+opts_parser = OptionParser.new do |opts|
+  opts.on("-c", "--config FILE") { |file| config_file = file }
+end
+
+opts_parser.parse!
+
+unless config_file
+  puts opts_parser
+  exit(1)
+end
+
+@config = YAML.load_file(config_file)
+
+module ConsoleHelpers
+  def cpi
+    @cpi ||= Bosh::AwsCloud::Cloud.new(@config)
+  end
+
+  def ec2
+    cpi.ec2
+  end
+
+  def registry
+    cpi.registry
+  end
+end
+
+cloud_config = OpenStruct.new(:logger => Logger.new(STDOUT))
+
+Bosh::Clouds::Config.configure(cloud_config)
+
+include ConsoleHelpers
+
+begin
+  require "ruby-debug"
+  puts "=> Debugger enabled"
+rescue LoadError
+  puts "=> ruby-debug not found, debugger disabled"
+end
+
+puts "=> Welcome to BOSH AWS CPI console"
+puts "You can use 'cpi' to access CPI methods"
+
+IRB.start
+
+
+

data/lib/cloud/aws/cloud.rb

@@ -11,13 +11,15 @@ module Bosh::AwsCloud
     METADATA_TIMEOUT = 5 # seconds
     DEVICE_POLL_TIMEOUT = 60 # seconds
 
-    DEFAULT_AKI = "aki-825ea7eb"
+    DEFAULT_AKI = "aki-b4aa75dd"
+    DEFAULT_ROOT_DEVICE_NAME = "/dev/sda1"
 
     # UBUNTU_10_04_32_BIT_US_EAST_EBS = "ami-3e9b4957"
     # UBUNTU_10_04_32_BIT_US_EAST = "ami-809a48e9"
 
     attr_reader :ec2
     attr_reader :registry
+    attr_accessor :logger
 
     ##
     # Initialize BOSH AWS CPI
@@ -94,15 +96,13 @@ module Bosh::AwsCloud
           }
         }
 
-        if disk_locality
-          # TODO: use as hint for availability zones
-          @logger.debug("Disk locality is ignored by AWS CPI")
-        end
-
         security_groups =
           network_configurator.security_groups(@default_security_groups)
         @logger.debug("using security groups: #{security_groups.join(', ')}")
 
+        response = @ec2.client.describe_images(:image_ids => [stemcell_id])
+        root_device_name = response.images_set.first.root_device_name
+
         instance_params = {
           :image_id => stemcell_id,
           :count => 1,
@@ -112,23 +112,20 @@ module Bosh::AwsCloud
           :user_data => Yajl::Encoder.encode(user_data)
         }
 
-        availability_zone = resource_pool["availability_zone"]
-        if availability_zone
-          instance_params[:availability_zone] = availability_zone
-        end
+        instance_params[:availability_zone] =
+          select_availability_zone(disk_locality,
+          resource_pool["availability_zone"])
 
         @logger.info("Creating new instance...")
         instance = @ec2.instances.create(instance_params)
-        state = instance.status
 
-        @logger.info("Creating new instance `#{instance.id}', " \
-                     "state is `#{state}'")
-
-        wait_resource(instance, state, :running)
+        @logger.info("Creating new instance `#{instance.id}'")
+        wait_resource(instance, :running)
 
         network_configurator.configure(@ec2, instance)
 
-        settings = initial_agent_settings(agent_id, network_spec, environment)
+        settings = initial_agent_settings(agent_id, network_spec, environment,
+                                          root_device_name)
         @registry.update_settings(instance.id, settings)
 
         instance.id
@@ -137,22 +134,23 @@ module Bosh::AwsCloud
 
     ##
     # Terminates EC2 instance and waits until it reports as terminated
-    # @param [String] vm_id Running instance id
+    # @param [String] instance_id Running instance id
     def delete_vm(instance_id)
       with_thread_name("delete_vm(#{instance_id})") do
         instance = @ec2.instances[instance_id]
 
         instance.terminate
-        state = instance.status
-
-        # TODO: should this be done before or after deleting VM?
-        @logger.info("Deleting instance settings for `#{instance.id}'")
-        @registry.delete_settings(instance.id)
 
-        @logger.info("Deleting instance `#{instance.id}', " \
-                     "state is `#{state}'")
-
-        wait_resource(instance, state, :terminated)
+        begin
+          # TODO: should this be done before or after deleting VM?
+          @logger.info("Deleting instance settings for `#{instance.id}'")
+          @registry.delete_settings(instance.id)
+
+          @logger.info("Deleting instance `#{instance.id}'")
+          wait_resource(instance, :terminated)
+        rescue AWS::EC2::Errors::InvalidInstanceID::NotFound
+          # It's OK, just means that instance has already been deleted
+        end
       end
     end
 
@@ -178,11 +176,11 @@ module Bosh::AwsCloud
           raise ArgumentError, "disk size needs to be an integer"
         end
 
-        if (size < 1024)
+        if size < 1024
           cloud_error("AWS CPI minimum disk size is 1 GiB")
         end
 
-        if (size > 1024 * 1000)
+        if size > 1024 * 1000
           cloud_error("AWS CPI maximum disk size is 1 TiB")
         end
 
@@ -199,12 +197,8 @@ module Bosh::AwsCloud
         }
 
         volume = @ec2.volumes.create(volume_params)
-        state = volume.state
-
-        @logger.info("Creating volume `#{volume.id}', " \
-                     "state is `#{state}'")
-
-        wait_resource(volume, state, :available)
+        @logger.info("Creating volume `#{volume.id}'")
+        wait_resource(volume, :available)
 
         volume.id
       end
@@ -227,12 +221,10 @@ module Bosh::AwsCloud
         volume.delete
 
         begin
-          state = volume.state
-          @logger.info("Deleting volume `#{volume.id}', " \
-                       "state is `#{state}'")
-
-          wait_resource(volume, state, :deleted)
+          @logger.info("Deleting volume `#{volume.id}'")
+          wait_resource(volume, :deleted)
         rescue AWS::EC2::Errors::InvalidVolume::NotFound
+          # It's OK, just means the volume has already been deleted
         end
 
         @logger.info("Volume `#{disk_id}' has been deleted")
@@ -310,32 +302,31 @@ module Bosh::AwsCloud
           ebs_volume = find_ebs_device(sd_name)
 
           # 2. Copy image to new EBS volume
-          Dir.mktmpdir do |tmp_dir|
-            @logger.info("Extracting stemcell to `#{tmp_dir}'")
-
-            unpack_image(tmp_dir, image_path)
-            copy_root_image(tmp_dir, ebs_volume)
-
-            # 3. Create snapshot and then an image using this snapshot
-            snapshot = volume.create_snapshot
-            wait_resource(snapshot, snapshot.status, :completed)
-
-            image_params = {
-              :name => "BOSH-#{generate_unique_name}",
-              :architecture => "x86_64",
-              :kernel_id => cloud_properties["kernel_id"] || DEFAULT_AKI,
-              :root_device_name => "/dev/sda",
-              :block_device_mappings => {
-                "/dev/sda" => { :snapshot_id => snapshot.id },
-                "/dev/sdb" => "ephemeral0"
-              }
+          @logger.info("Copying stemcell disk image to '#{ebs_volume}'")
+          copy_root_image(image_path, ebs_volume)
+
+          # 3. Create snapshot and then an image using this snapshot
+          snapshot = volume.create_snapshot
+          wait_resource(snapshot, :completed)
+
+          root_device_name = cloud_properties["root_device_name"] ||
+                             DEFAULT_ROOT_DEVICE_NAME
+
+          image_params = {
+            :name => "BOSH-#{generate_unique_name}",
+            :architecture => "x86_64", # TODO should this be configurable?
+            :kernel_id => cloud_properties["kernel_id"] || DEFAULT_AKI,
+            :root_device_name =>  root_device_name,
+            :block_device_mappings => {
+              "/dev/sda" => { :snapshot_id => snapshot.id },
+              "/dev/sdb" => "ephemeral0"
             }
+          }
 
-            image = @ec2.images.create(image_params)
-            wait_resource(image, image.state, :available, :state)
+          image = @ec2.images.create(image_params)
+          wait_resource(image, :available, :state)
 
-            image.id
-          end
+          image.id
         rescue => e
           # TODO: delete snapshot?
           @logger.error(e)
@@ -361,6 +352,32 @@ module Bosh::AwsCloud
       not_implemented(:validate_deployment)
     end
 
+    # Selects the availability zone to use from a list of disk volumes,
+    # resource pool availability zone (if any) and the default availability
+    # zone.
+    # @param [Hash] volumes volume ids to attach to the vm
+    # @param [String] resource_pool_az availability zone specified in
+    #   the resource pool (may be nil)
+    # @return [String] availability zone to use
+    def select_availability_zone(volumes, resource_pool_az)
+      if volumes && !volumes.empty?
+        disks = volumes.map { |vid| @ec2.volumes[vid] }
+        ensure_same_availability_zone(disks, resource_pool_az)
+        disks.first.availability_zone
+      else
+        resource_pool_az || DEFAULT_AVAILABILITY_ZONE
+      end
+    end
+
+    # ensure all supplied availability zones are the same
+    def ensure_same_availability_zone(disks, default)
+      zones = disks.map { |disk| disk.availability_zone }
+      zones << default if default
+      zones.uniq!
+      cloud_error "can't use multiple availability zones: %s" %
+        zones.join(", ") unless zones.size == 1 || zones.empty?
+    end
+
     private
 
     ##
@@ -377,7 +394,8 @@ module Bosh::AwsCloud
     # @param [Hash] network_spec Agent network spec
     # @param [Hash] environment
     # @return [Hash]
-    def initial_agent_settings(agent_id, network_spec, environment)
+    def initial_agent_settings(agent_id, network_spec, environment,
+                               root_device_name)
       settings = {
         "vm" => {
           "name" => "vm-#{generate_unique_name}"
@@ -385,7 +403,7 @@ module Bosh::AwsCloud
         "agent_id" => agent_id,
         "networks" => network_spec,
         "disks" => {
-          "system" => "/dev/sda",
+          "system" => root_device_name,
           "ephemeral" => "/dev/sdb",
           "persistent" => {}
         }
@@ -438,7 +456,9 @@ module Bosh::AwsCloud
     end
 
     def attach_ebs_volume(instance, volume)
-      device_names = Set.new(instance.block_device_mappings.keys)
+      # TODO once we upgrade the aws-sdk gem to > 1.3.9, we need to use:
+      # instance.block_device_mappings.to_hash.keys
+      device_names = Set.new(instance.block_device_mappings.to_hash.keys)
       new_attachment = nil
 
       ("f".."p").each do |char| # f..p is what console suggests
@@ -458,12 +478,9 @@ module Bosh::AwsCloud
         cloud_error("Instance has too many disks attached")
       end
 
-      state = new_attachment.status
+      @logger.info("Attaching `#{volume.id}' to `#{instance.id}'")
+      wait_resource(new_attachment, :attached)
 
-      @logger.info("Attaching `#{volume.id}' to #{instance.id}, " \
-                   "state is #{state}'")
-
-      wait_resource(new_attachment, state, :attached)
       device_name = new_attachment.device
 
       @logger.info("Attached `#{volume.id}' to `#{instance.id}', " \
@@ -473,7 +490,9 @@ module Bosh::AwsCloud
     end
 
     def detach_ebs_volume(instance, volume)
-      mappings = instance.block_device_mappings
+      # TODO once we upgrade the aws-sdk gem to > 1.3.9, we need to use:
+      # instance.block_device_mappings.to_hash.keys
+      mappings = instance.block_device_mappings.to_hash
 
       device_map = mappings.inject({}) do |hash, (device_name, attachment)|
         hash[attachment.volume.id] = device_name
@@ -486,40 +505,51 @@ module Bosh::AwsCloud
       end
 
       attachment = volume.detach_from(instance, device_map[volume.id])
-      state = attachment.status
-
-      @logger.info("Detaching `#{volume.id}' from `#{instance.id}', " \
-                   "state is #{state}'")
+      @logger.info("Detaching `#{volume.id}' from `#{instance.id}'")
 
       begin
-        wait_resource(attachment, state, :detached)
+        wait_resource(attachment, :detached)
       rescue AWS::Core::Resource::NotFound
-        # It's OK, just means attachment is gone when we're asking for state
+        # It's OK, just means attachment is gone by now
       end
     end
 
-    def unpack_image(tmp_dir, image_path)
-      output = `tar -C #{tmp_dir} -xzf #{image_path} 2>&1`
-      if $?.exitstatus != 0
-        cloud_error("Failed to unpack stemcell root image" \
-                    "tar exit status #{$?.exitstatus}: #{output}")
+    # This method tries to execute the helper script stemcell-copy
+    # as root using sudo, since it needs to write to the ebs_volume.
+    # If stemcell-copy isn't available, it falls back to writing directly
+    # to the device, which is used in the micro bosh deployer.
+    # The stemcell-copy script must be in the PATH of the user running
+    # the director, and needs sudo privileges to execute without
+    # password.
+    def copy_root_image(image_path, ebs_volume)
+      path = ENV["PATH"]
+
+      if stemcell_copy = has_stemcell_copy(path)
+        @logger.debug("copying stemcell using stemcell-copy script")
+        # note that is is a potentially dangerous operation, but as the
+        # stemcell-copy script sets PATH to a sane value this is safe
+        out = `sudo #{stemcell_copy} #{image_path} #{ebs_volume} 2>&1`
+      else
+        @logger.info("falling back to using dd to copy stemcell")
+        out = `tar -xzf #{image_path} -O root.img | dd of=#{ebs_volume} 2>&1`
       end
 
-      root_image = File.join(tmp_dir, "root.img")
-      unless File.exists?(root_image)
-        cloud_error("Root image is missing from stemcell archive")
+      unless $?.exitstatus == 0
+        cloud_error("Unable to copy stemcell root image, " \
+                    "exit status #{$?.exitstatus}: #{out}")
       end
+
+      @logger.debug("stemcell copy output:\n#{out}")
     end
 
-    def copy_root_image(dir, ebs_volume)
-      Dir.chdir(dir) do
-        dd_out = `dd if=root.img of=#{ebs_volume} 2>&1`
-        if $?.exitstatus != 0
-          cloud_error("Unable to copy stemcell root image, " \
-                      "dd exit status #{$?.exitstatus}: " \
-                      "#{dd_out}")
-        end
+    # checks if the stemcell-copy script can be found in
+    # the current PATH
+    def has_stemcell_copy(path)
+      path.split(":").each do |dir|
+        stemcell_copy = File.join(dir, "stemcell-copy")
+        return stemcell_copy if File.exist?(stemcell_copy)
       end
+      nil
     end
 
     def find_ebs_device(sd_name)
@@ -554,20 +584,13 @@ module Bosh::AwsCloud
       # N.B. This will only work with ebs-store instances,
       # as instance-store instances don't support stop/start.
       instance.stop
-      state = instance.status
 
-      @logger.info("Stopping instance `#{instance.id}', " \
-                   "state is `#{state}'")
-
-      wait_resource(instance, state, :stopped)
+      @logger.info("Stopping instance `#{instance.id}'")
+      wait_resource(instance, :stopped)
 
       instance.start
-      state = instance.status
-
-      @logger.info("Starting instance `#{instance.id}', " \
-                   "state is `#{state}'")
-
-      wait_resource(instance, state, :running)
+      @logger.info("Starting instance `#{instance.id}'")
+      wait_resource(instance, :running)
     end
 
     ##

data/lib/cloud/aws/helpers.rb

@@ -4,7 +4,7 @@ module Bosh::AwsCloud
 
   module Helpers
 
-    DEFAULT_TIMEOUT = 3600
+    DEFAULT_TIMEOUT = 3600 # seconds
 
     ##
     # Raises CloudError exception
@@ -16,43 +16,61 @@ module Bosh::AwsCloud
       raise Bosh::Clouds::CloudError, message
     end
 
-    def wait_resource(resource, start_state,
-                      target_state, state_method = :status,
+    def wait_resource(resource, target_state, state_method = :status,
                       timeout = DEFAULT_TIMEOUT)
 
       started_at = Time.now
-      state = resource.send(state_method)
+      failures = 0
       desc = resource.to_s
 
-      while state == start_state && state != target_state
+      loop do
         duration = Time.now - started_at
 
         if duration > timeout
-          cloud_error("Timed out waiting for #{desc} " \
-                      "to be #{target_state}")
+          cloud_error("Timed out waiting for #{desc} to be #{target_state}")
         end
 
         if @logger
-          @logger.debug("Waiting for #{desc} " \
-                        "to be #{target_state} (#{duration})")
+          @logger.debug("Waiting for #{desc} to be #{target_state} " \
+                        "(#{duration}s)")
         end
 
-        sleep(1)
+        begin
+          state = resource.send(state_method)
+        rescue AWS::EC2::Errors::InvalidAMIID::NotFound,
+               AWS::EC2::Errors::InvalidInstanceID::NotFound => e
+          # ugly workaround for AWS race conditions:
+          # 1) sometimes when we upload a stemcell and proceed to create a VM
+          #    from it, AWS reports that the AMI is missing
+          # 2) sometimes when we create a new EC2 instance, AWS reports that
+          #    the instance it returns is missing
+          # in both cases we just wait a little and retry...
+          raise e if failures > 3
+          failures += 1
+          @logger.error("#{e.message}: #{desc}")
+          sleep(1)
+          next
+        end
+
+        # This is not a very strong convention, but some resources
+        # have 'error' and 'failed' states, we probably don't want to keep
+        # waiting if we're in these states. Alternatively we could introduce a
+        # set of 'loop breaker' states but that doesn't seem very helpful
+        # at the moment
+        if state == :error || state == :failed
+          cloud_error("#{desc} state is #{state}, expected #{target_state}")
+        end
+
+        break if state == target_state
 
-        state = resource.send(state_method)
+        sleep(1)
       end
 
-      if state == target_state
-        if @logger
-          @logger.info("#{desc} is #{target_state} " \
-                       "after #{Time.now - started_at}s")
-        end
-      else
-        cloud_error("#{desc} is #{state}, " \
-                    "expected to be #{target_state}")
+      if @logger
+        total = Time.now - started_at
+        @logger.info("#{desc} is now #{target_state}, took #{total}s")
       end
     end
   end
-
 end
 

data/lib/cloud/aws/version.rb

@@ -2,6 +2,6 @@
 
 module Bosh
   module AwsCloud
-    VERSION = "0.3.3"
+    VERSION = "0.5.0"
   end
 end

data/spec/assets/stemcell-copy

@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# This script runs as root through sudo without the need for a password,
+# so it needs to make sure it can't be abused.
+#
+
+# make sure we have a secure PATH
+PATH=/bin:/usr/bin
+export PATH
+
+if [ $# -ne 1 ]; then
+  echo "usage: $0 <block device>"
+  exit 1
+fi
+
+OUTPUT="$1"
+
+# TODO perhaps this should be more restrictive?
+echo ${OUTPUT} | egrep '^/dev/[a-z0-9]+$' > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+  echo "ERROR: illegal device: ${OUTPUT}"
+  exit 1
+fi
+
+if [ ! -b ${OUTPUT} ]; then
+  echo "ERROR: missing device: ${OUTPUT}"
+  exit 1
+fi
+
+# copy image to block device with 1 MB block size
+dd if=root.img if=${OUTPUT} bs=1M

data/spec/integration/cpi_test.rb

@@ -0,0 +1,78 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require File.expand_path("../../spec_helper", __FILE__)
+
+require "tempfile"
+
+describe Bosh::AwsCloud::Cloud do
+
+  before(:each) do
+    unless ENV["CPI_CONFIG_FILE"]
+      raise "Please provide CPI_CONFIG_FILE environment variable"
+    end
+    @config = YAML.load_file(ENV["CPI_CONFIG_FILE"])
+    @logger = Logger.new(STDOUT)
+  end
+
+  let(:cpi) do
+    cpi = Bosh::AwsCloud::Cloud.new(@config)
+    cpi.logger = @logger
+
+    # As we inject the configuration file from the outside, we don't care
+    # about spinning up the registry ourselves. However we don't want to bother
+    # EC2 at all if registry is not working, so just in case we perform a test
+    # health check against whatever has been provided.
+    cpi.registry.update_settings("foo", { "bar" => "baz" })
+    cpi.registry.read_settings("foo").should == { "bar" => "baz"}
+
+    cpi
+  end
+
+  it "exercises a VM lifecycle" do
+    instance_id = cpi.create_vm(
+      "agent-007", "ami-809a48e9",
+      { "instance_type" => "m1.small" },
+      { "default" => { "type" => "dynamic" }},
+      [], { "key" => "value" })
+
+    instance_id.should_not be_nil
+
+    settings = cpi.registry.read_settings(instance_id)
+    settings["vm"].should be_a(Hash)
+    settings["vm"]["name"].should_not be_nil
+    settings["agent_id"].should == "agent-007"
+    settings["networks"].should == { "default" => { "type" => "dynamic" }}
+    settings["disks"].should == {
+      "system" => "/dev/sda",
+      "ephemeral" => "/dev/sdb",
+      "persistent" => {}
+    }
+
+    settings["env"].should == { "key" => "value" }
+
+    volume_id = cpi.create_disk(2048)
+    volume_id.should_not be_nil
+
+    cpi.attach_disk(instance_id, volume_id)
+    settings = cpi.registry.read_settings(instance_id)
+    settings["disks"]["persistent"].should == { volume_id => "/dev/sdf" }
+
+    cpi.detach_disk(instance_id, volume_id)
+    settings = cpi.registry.read_settings(instance_id)
+    settings["disks"]["persistent"].should == {}
+
+    # TODO: test configure_networks (need an elastic IP at hand for that)
+
+    cpi.delete_vm(instance_id)
+    cpi.delete_disk(volume_id)
+
+    # Test below would fail: EC2 still reports the instance as 'terminated'
+    # for some time.
+    # cpi.ec2.instances[instance_id].should be_nil
+
+    expect {
+      cpi.registry.read_settings(instance_id)
+    }.to raise_error(/HTTP 404/)
+  end
+
+end

data/spec/spec_helper.rb

@@ -25,6 +25,20 @@ Bosh::Clouds::Config.configure(aws_config)
 MOCK_AWS_ACCESS_KEY_ID = "foo"
 MOCK_AWS_SECRET_ACCESS_KEY = "bar"
 
+def internal_to(*args, &block)
+  example = describe *args, &block
+  klass = args[0]
+  if klass.is_a? Class
+    saved_private_instance_methods = klass.private_instance_methods
+    example.before do
+      klass.class_eval { public *saved_private_instance_methods }
+    end
+    example.after do
+      klass.class_eval { private *saved_private_instance_methods }
+    end
+  end
+end
+
 def mock_cloud_options
   {
     "aws" => {

data/spec/unit/attach_disk_spec.rb

@@ -23,8 +23,7 @@ describe Bosh::AwsCloud::Cloud do
 
     instance.should_receive(:block_device_mappings).and_return({})
 
-    attachment.should_receive(:status).and_return(:attaching)
-    cloud.should_receive(:wait_resource).with(attachment, :attaching, :attached)
+    cloud.should_receive(:wait_resource).with(attachment, :attached)
 
     old_settings = { "foo" => "bar" }
     new_settings = {
@@ -61,8 +60,7 @@ describe Bosh::AwsCloud::Cloud do
     volume.should_receive(:attach_to).
       with(instance, "/dev/sdh").and_return(attachment)
 
-    attachment.should_receive(:status).and_return(:attaching)
-    cloud.should_receive(:wait_resource).with(attachment, :attaching, :attached)
+    cloud.should_receive(:wait_resource).with(attachment, :attached)
 
     old_settings = { "foo" => "bar" }
     new_settings = {
@@ -99,8 +97,7 @@ describe Bosh::AwsCloud::Cloud do
     volume.should_receive(:attach_to).
       with(instance, "/dev/sdh").and_return(attachment)
 
-    attachment.should_receive(:status).and_return(:attaching)
-    cloud.should_receive(:wait_resource).with(attachment, :attaching, :attached)
+    cloud.should_receive(:wait_resource).with(attachment, :attached)
 
     old_settings = { "foo" => "bar" }
     new_settings = {

data/spec/unit/cloud_spec.rb

@@ -13,4 +13,20 @@ describe Bosh::AwsCloud::Cloud do
 
   end
 
+  internal_to Bosh::AwsCloud::Cloud do
+
+    it "should not find stemcell-copy" do
+      cloud = Bosh::Clouds::Provider.create(:aws, mock_cloud_options)
+      cloud.has_stemcell_copy("/usr/bin:/usr/sbin").should be_nil
+    end
+
+    it "should find stemcell-copy" do
+      cloud = Bosh::Clouds::Provider.create(:aws, mock_cloud_options)
+      path = ENV["PATH"]
+      path += ":#{File.expand_path('../../assets', __FILE__)}"
+      cloud.has_stemcell_copy(path).should_not be_nil
+    end
+
+  end
+
 end

data/spec/unit/create_disk_spec.rb

@@ -16,8 +16,7 @@ describe Bosh::AwsCloud::Cloud do
       ec2.volumes.should_receive(:create).with(disk_params).and_return(volume)
     end
 
-    volume.should_receive(:state).and_return(:creating)
-    cloud.should_receive(:wait_resource).with(volume, :creating, :available)
+    cloud.should_receive(:wait_resource).with(volume, :available)
 
     cloud.create_disk(2048).should == "v-foobar"
   end
@@ -34,8 +33,7 @@ describe Bosh::AwsCloud::Cloud do
       ec2.volumes.should_receive(:create).with(disk_params).and_return(volume)
     end
 
-    volume.should_receive(:state).and_return(:creating)
-    cloud.should_receive(:wait_resource).with(volume, :creating, :available)
+    cloud.should_receive(:wait_resource).with(volume, :available)
 
     cloud.create_disk(2049)
   end
@@ -67,8 +65,7 @@ describe Bosh::AwsCloud::Cloud do
       ec2.instances.stub(:[]).with("i-test").and_return(instance)
     end
 
-    volume.should_receive(:state).and_return(:creating)
-    cloud.should_receive(:wait_resource).with(volume, :creating, :available)
+    cloud.should_receive(:wait_resource).with(volume, :available)
 
     cloud.create_disk(1024, "i-test")
   end

data/spec/unit/create_stemcell_spec.rb

@@ -27,8 +27,8 @@ describe Bosh::AwsCloud::Cloud do
       image_params = {
         :name => "BOSH-#{unique_name}",
         :architecture => "x86_64",
-        :kernel_id => "aki-825ea7eb",
-        :root_device_name => "/dev/sda",
+        :kernel_id => "aki-b4aa75dd",
+        :root_device_name => "/dev/sda1",
         :block_device_mappings => {
           "/dev/sda" => { :snapshot_id => "s-baz" },
           "/dev/sdb" => "ephemeral0"
@@ -68,35 +68,24 @@ describe Bosh::AwsCloud::Cloud do
       volume.should_receive(:attach_to).with(current_instance, "/dev/sdh").
         and_return(attachment)
 
-      attachment.should_receive(:status).and_return(:attaching)
-      cloud.should_receive(:wait_resource).
-        with(attachment, :attaching, :attached)
+      cloud.should_receive(:wait_resource).with(attachment, :attached)
 
       cloud.stub(:sleep)
 
       File.stub(:blockdev?).with("/dev/sdh").and_return(false, false, false)
       File.stub(:blockdev?).with("/dev/xvdh").and_return(false, false, true)
 
-      Dir.should_receive(:mktmpdir).and_yield(@tmp_dir)
-
-      cloud.should_receive(:unpack_image).with(@tmp_dir, "/tmp/foo")
-      cloud.should_receive(:copy_root_image).with(@tmp_dir, "/dev/xvdh")
+      cloud.should_receive(:copy_root_image).with("/tmp/foo", "/dev/xvdh")
 
       volume.should_receive(:create_snapshot).and_return(snapshot)
-      snapshot.should_receive(:status).and_return(:in_progress)
-      cloud.should_receive(:wait_resource).
-        with(snapshot, :in_progress, :completed)
+      cloud.should_receive(:wait_resource).with(snapshot, :completed)
 
-      image.should_receive(:state).and_return(:creating)
-      cloud.should_receive(:wait_resource).with(image,
-                                                :creating, :available, :state)
+      cloud.should_receive(:wait_resource).with(image, :available, :state)
 
       volume.should_receive(:detach_from).with(current_instance, "/dev/sdh").
         and_return(attachment)
 
-      attachment.should_receive(:status).and_return(:detaching)
-      cloud.should_receive(:wait_resource).
-        with(attachment, :detaching, :detached)
+      cloud.should_receive(:wait_resource).with(attachment, :detached)
 
       cloud.should_receive(:delete_disk).with("v-foo")
 

data/spec/unit/create_vm_spec.rb

@@ -12,7 +12,7 @@ describe Bosh::AwsCloud::Cloud, "create_vm" do
       "agent_id" => "agent-id",
       "networks" => { "network_a" => network_spec },
       "disks" => {
-        "system" => "/dev/sda",
+        "system" => "/dev/sda1",
         "ephemeral" => "/dev/sdb",
         "persistent" => {}
       },
@@ -24,6 +24,11 @@ describe Bosh::AwsCloud::Cloud, "create_vm" do
     }
   end
 
+  def fake_image_set
+    image = double("image", :root_device_name => "/dev/sda1")
+    double("images_set", :images_set => [image])
+  end
+
   def ec2_params(user_data, security_groups=[])
     {
       :image_id => "sc-id",
@@ -52,16 +57,17 @@ describe Bosh::AwsCloud::Cloud, "create_vm" do
     instance = double("instance",
                       :id => "i-test",
                       :elastic_ip => nil)
+    client = double("client", :describe_images => fake_image_set)
 
     cloud = mock_cloud do |ec2|
       ec2.instances.should_receive(:create).
         with(ec2_params(user_data)).
         and_return(instance)
+      ec2.should_receive(:client).and_return(client)
     end
 
-    instance.should_receive(:status).and_return(:pending)
     cloud.should_receive(:generate_unique_name).and_return(unique_name)
-    cloud.should_receive(:wait_resource).with(instance, :pending, :running)
+    cloud.should_receive(:wait_resource).with(instance, :running)
     @registry.should_receive(:update_settings)
       .with("i-test", agent_settings(unique_name))
 
@@ -85,6 +91,7 @@ describe Bosh::AwsCloud::Cloud, "create_vm" do
     instance = double("instance",
                       :id => "i-test",
                       :elastic_ip => nil)
+    client = double("client", :describe_images => fake_image_set)
 
     security_groups = %w[foo bar]
     network_spec = dynamic_network_spec
@@ -96,11 +103,11 @@ describe Bosh::AwsCloud::Cloud, "create_vm" do
       ec2.instances.should_receive(:create).
         with(ec2_params(user_data, security_groups)).
         and_return(instance)
+      ec2.should_receive(:client).and_return(client)
     end
 
-    instance.should_receive(:status).and_return(:pending)
     cloud.should_receive(:generate_unique_name).and_return(unique_name)
-    cloud.should_receive(:wait_resource).with(instance, :pending, :running)
+    cloud.should_receive(:wait_resource).with(instance, :running)
     @registry.should_receive(:update_settings)
       .with("i-test", agent_settings(unique_name, network_spec))
 
@@ -116,14 +123,15 @@ describe Bosh::AwsCloud::Cloud, "create_vm" do
     instance = double("instance",
                       :id => "i-test",
                       :elastic_ip => nil)
+    client = double("client", :describe_images => fake_image_set)
 
     cloud = mock_cloud do |ec2|
       ec2.instances.should_receive(:create).and_return(instance)
+      ec2.should_receive(:client).and_return(client)
     end
 
-    instance.should_receive(:status).and_return(:pending)
     instance.should_receive(:associate_elastic_ip).with("10.0.0.1")
-    cloud.should_receive(:wait_resource).with(instance, :pending, :running)
+    cloud.should_receive(:wait_resource).with(instance, :running)
     @registry.should_receive(:update_settings)
 
     vm_id = cloud.create_vm("agent-id", "sc-id",
@@ -131,4 +139,56 @@ describe Bosh::AwsCloud::Cloud, "create_vm" do
                             combined_network_spec)
   end
 
+  def volume(zone)
+    vol = double("volume")
+    vol.stub(:availability_zone).and_return(zone)
+    vol
+  end
+
+  describe "#select_availability_zone" do
+
+    it "should select the default availability_zone when all values are nil" do
+      cloud = mock_cloud
+      cloud.select_availability_zone(nil, nil).should ==
+        Bosh::AwsCloud::Cloud::DEFAULT_AVAILABILITY_ZONE
+    end
+
+    it "should select the zone from a list of disks" do
+      cloud = mock_cloud do |ec2|
+        ec2.volumes.stub(:[]).and_return(volume("foo"), volume("foo"))
+      end
+      cloud.select_availability_zone(%w[cid1 cid2], nil).should == "foo"
+    end
+
+    it "should select the zone from a list of disks and a default" do
+      cloud = mock_cloud do |ec2|
+        ec2.volumes.stub(:[]).and_return(volume("foo"), volume("foo"))
+      end
+      cloud.select_availability_zone(%w[cid1 cid2], "foo").should == "foo"
+    end
+  end
+
+  describe "#ensure_same_availability_zone" do
+    it "should raise an error when the zones differ" do
+      cloud = mock_cloud
+      lambda {
+        cloud.ensure_same_availability_zone([volume("foo"), volume("bar")], nil)
+      }.should raise_error Bosh::Clouds::CloudError
+    end
+
+    it "should raise an error when the zones differ" do
+      cloud = mock_cloud
+      lambda {
+        cloud.ensure_same_availability_zone([volume("foo"), volume("bar")], "foo")
+      }.should raise_error Bosh::Clouds::CloudError
+    end
+
+    it "should raise an error when the zones differ" do
+      cloud = mock_cloud
+      lambda {
+        cloud.ensure_same_availability_zone([volume("foo"), volume("foo")], "bar")
+      }.should raise_error Bosh::Clouds::CloudError
+    end
+  end
+
 end

data/spec/unit/delete_disk_spec.rb

@@ -11,9 +11,9 @@ describe Bosh::AwsCloud::Cloud do
       ec2.volumes.stub(:[]).with("v-foo").and_return(volume)
     end
 
-    volume.should_receive(:state).and_return(:available, :deleting)
+    volume.should_receive(:state).and_return(:available)
     volume.should_receive(:delete)
-    cloud.should_receive(:wait_resource).with(volume, :deleting, :deleted)
+    cloud.should_receive(:wait_resource).with(volume, :deleted)
 
     cloud.delete_disk("v-foo")
   end

data/spec/unit/delete_vm_spec.rb

@@ -16,8 +16,7 @@ describe Bosh::AwsCloud::Cloud do
     end
 
     instance.should_receive(:terminate)
-    instance.should_receive(:status).and_return(:deleting)
-    cloud.should_receive(:wait_resource).with(instance, :deleting, :terminated)
+    cloud.should_receive(:wait_resource).with(instance, :terminated)
 
     @registry.should_receive(:delete_settings).with("i-foobar")
 

data/spec/unit/detach_disk_spec.rb

@@ -30,8 +30,7 @@ describe Bosh::AwsCloud::Cloud do
     volume.should_receive(:detach_from).
       with(instance, "/dev/sdf").and_return(attachment)
 
-    attachment.should_receive(:status).and_return(:detaching)
-    cloud.should_receive(:wait_resource).with(attachment, :detaching, :detached)
+    cloud.should_receive(:wait_resource).with(attachment, :detached)
 
     old_settings = {
       "foo" => "bar",

data/spec/unit/helpers_spec.rb

@@ -11,7 +11,7 @@ describe Bosh::AwsCloud::Helpers do
     cloud.stub(:sleep)
 
     lambda {
-      cloud.wait_resource(resource, :start, :stop, :status, 0.1)
+      cloud.wait_resource(resource, :stop, :status, 0.1)
     }.should raise_error Bosh::Clouds::CloudError, /Timed out/
   end
 
@@ -19,11 +19,11 @@ describe Bosh::AwsCloud::Helpers do
     cloud = mock_cloud
 
     resource = double("resource")
-    resource.stub(:status).and_return(:start, :stop)
+    resource.stub(:status).and_return(:start, :stopping, :stopping, :stop)
     cloud.stub(:sleep)
 
     lambda {
-      cloud.wait_resource(resource, :start, :stop, :status, 0.1)
+      cloud.wait_resource(resource, :stop, :status, 0.1)
     }.should_not raise_error Bosh::Clouds::CloudError
   end
 
@@ -35,8 +35,8 @@ describe Bosh::AwsCloud::Helpers do
     cloud.stub(:sleep)
 
     lambda {
-      cloud.wait_resource(resource, :started, :stopped, :status, 0.1)
+      cloud.wait_resource(resource, :stopped, :status, 0.1)
     }.should raise_error Bosh::Clouds::CloudError,
-                         /is failed, expected to be stopped/
+                         /is failed, expected stopped/
   end
 end

data/spec/unit/reboot_vm_spec.rb

@@ -25,14 +25,12 @@ describe Bosh::AwsCloud::Cloud do
   it "hard reboots an EC2 instance" do
     # N.B. This requires ebs-store instance
     @instance.should_receive(:stop).ordered
-    @instance.should_receive(:status).ordered.and_return(:stopping)
     @cloud.should_receive(:wait_resource).
-      with(@instance, :stopping, :stopped).ordered
+      with(@instance, :stopped).ordered
 
     @instance.should_receive(:start)
-    @instance.should_receive(:status).and_return(:starting)
     @cloud.should_receive(:wait_resource).ordered.
-      with(@instance, :starting, :running)
+      with(@instance, :running)
 
     @cloud.send(:hard_reboot, @instance)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bosh_aws_cpi
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-08 00:00:00.000000000 Z
+date: 2012-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.3
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -109,10 +109,12 @@ dependencies:
         version: 0.8.2
 description: BOSH AWS CPI
 email: support@vmware.com
-executables: []
+executables:
+- bosh_aws_console
 extensions: []
 extra_rdoc_files: []
 files:
+- bin/bosh_aws_console
 - lib/bosh_aws_cpi.rb
 - lib/cloud/aws.rb
 - lib/cloud/aws/cloud.rb
@@ -125,6 +127,8 @@ files:
 - lib/cloud/aws/vip_network.rb
 - README
 - Rakefile
+- spec/assets/stemcell-copy
+- spec/integration/cpi_test.rb
 - spec/spec_helper.rb
 - spec/unit/attach_disk_spec.rb
 - spec/unit/cloud_spec.rb
@@ -154,7 +158,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3401725189883691334
+      hash: -2403392629027305829
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -163,14 +167,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3401725189883691334
+      hash: -2403392629027305829
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.21
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: BOSH AWS CPI
 test_files:
+- spec/assets/stemcell-copy
+- spec/integration/cpi_test.rb
 - spec/spec_helper.rb
 - spec/unit/attach_disk_spec.rb
 - spec/unit/cloud_spec.rb