bosh_aws_cpi 0.7.0 → 1.5.0.pre.1113

data/lib/cloud/aws/stemcell_creator.rb

@@ -0,0 +1,114 @@
+module Bosh::AwsCloud
+  class StemcellCreator
+    include Bosh::Exec
+    include Helpers
+
+    attr_reader :region, :stemcell_properties
+    attr_reader :volume, :ebs_volume, :image_path
+
+    def initialize(region, stemcell_properties)
+      @region = region
+      @stemcell_properties = stemcell_properties
+    end
+
+    def create(volume, ebs_volume, image_path)
+      @volume = volume
+      @ebs_volume = ebs_volume
+      @image_path = image_path
+
+      copy_root_image
+
+      snapshot = volume.create_snapshot
+      ResourceWait.for_snapshot(snapshot: snapshot, state: :completed)
+
+      params = image_params(snapshot.id)
+      image = region.images.create(params)
+      ResourceWait.for_image(image: image, state: :available)
+
+      TagManager.tag(image, 'Name', params[:description]) if params[:description]
+
+      Stemcell.new(region, image)
+    end
+
+    def fake?
+      stemcell_properties.has_key?('ami')
+    end
+
+    def fake
+      id = stemcell_properties['ami'][region.name]
+
+      raise Bosh::Clouds::CloudError, "Stemcell does not contain an AMI for this region (#{region.name})" unless id
+
+      Stemcell.find(region, id)
+    end
+
+    # This method tries to execute the helper script stemcell-copy
+    # as root using sudo, since it needs to write to the ebs_volume.
+    # If stemcell-copy isn't available, it falls back to writing directly
+    # to the device, which is used in the micro bosh deployer.
+    # The stemcell-copy script must be in the PATH of the user running
+    # the director, and needs sudo privileges to execute without
+    # password.
+    #
+    def copy_root_image
+      stemcell_copy = find_in_path("stemcell-copy")
+
+      if stemcell_copy
+        logger.debug("copying stemcell using stemcell-copy script")
+        # note that is is a potentially dangerous operation, but as the
+        # stemcell-copy script sets PATH to a sane value this is safe
+        command = "sudo -n #{stemcell_copy} #{image_path} #{ebs_volume} 2>&1"
+      else
+        logger.info("falling back to using included copy stemcell")
+        included_stemcell_copy = File.expand_path("../../../../scripts/stemcell-copy.sh", __FILE__)
+        command = "sudo -n #{included_stemcell_copy} #{image_path} #{ebs_volume} 2>&1"
+      end
+
+      result = sh(command)
+
+      logger.debug("stemcell copy output:\n#{result.output}")
+    rescue Bosh::Exec::Error => e
+      raise Bosh::Clouds::CloudError, "Unable to copy stemcell root image: #{e.message}"
+    end
+
+    # checks if the stemcell-copy script can be found in
+    # the current PATH
+    def find_in_path(command, path=ENV["PATH"])
+      path.split(":").each do |dir|
+        stemcell_copy = File.join(dir, command)
+        return stemcell_copy if File.exist?(stemcell_copy)
+      end
+      nil
+    end
+
+    def image_params(snapshot_id)
+      root_device_name = stemcell_properties["root_device_name"]
+      architecture = stemcell_properties["architecture"]
+
+      aki = AKIPicker.new(region).pick(architecture, root_device_name)
+
+      params = {
+          :name => "BOSH-#{SecureRandom.uuid}",
+          :architecture => architecture,
+          :kernel_id => aki,
+          :root_device_name => root_device_name,
+          :block_device_mappings => {
+              "/dev/sda" => {:snapshot_id => snapshot_id},
+              "/dev/sdb" => "ephemeral0"
+          }
+      }
+
+      # old stemcells doesn't have name & version
+      if stemcell_properties["name"] && stemcell_properties["version"]
+        name = "#{stemcell_properties['name']} #{stemcell_properties['version']}"
+        params[:description] = name
+      end
+
+      params
+    end
+
+    def logger
+      Bosh::Clouds::Config.logger
+    end
+  end
+end

data/lib/cloud/aws/tag_manager.rb

@@ -0,0 +1,30 @@
+module Bosh::AwsCloud
+  class TagManager
+
+    MAX_TAG_KEY_LENGTH = 127
+    MAX_TAG_VALUE_LENGTH = 255
+
+    # Add a tag to something, make sure that the tag conforms to the
+    # AWS limitation of 127 character key and 255 character value
+    def self.tag(taggable, key, value)
+      return if key.nil? || value.nil?
+      trimmed_key = key.to_s.slice(0, MAX_TAG_KEY_LENGTH)
+      trimmed_value = value.to_s.slice(0, MAX_TAG_VALUE_LENGTH)
+      taggable.add_tag(trimmed_key, :value => trimmed_value)
+    rescue AWS::EC2::Errors::InvalidParameterValue => e
+      logger.error("could not tag #{taggable.id}: #{e.message}")
+    rescue AWS::EC2::Errors::InvalidAMIID::NotFound,
+        AWS::EC2::Errors::InvalidInstanceID::NotFound=> e
+      # Due to the AWS eventual consistency, the taggable might not
+      # be there, even though we previous have waited until it is,
+      # so we wait again...
+      logger.warn("tagged object doesn't exist: #{taggable.id}")
+      sleep(1)
+      retry
+    end
+
+    def self.logger
+      Bosh::Clouds::Config.logger
+    end
+  end
+end

data/lib/cloud/aws/version.rb

@@ -2,6 +2,6 @@
 
 module Bosh
   module AwsCloud
-    VERSION = "0.7.0"
+    VERSION = '1.5.0.pre.1113'
   end
 end

data/lib/cloud/aws/vip_network.rb

@@ -1,8 +1,7 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
 module Bosh::AwsCloud
-  ##
-  #
+
   class VipNetwork < Network
 
     ##
@@ -24,18 +23,21 @@ module Bosh::AwsCloud
         cloud_error("No IP provided for vip network `#{@name}'")
       end
 
+      elastic_ip = ec2.elastic_ips[@ip]
       @logger.info("Associating instance `#{instance.id}' " \
-                   "with elastic IP `#{@ip}'")
+                   "with elastic IP `#{elastic_ip}'")
 
       # New elastic IP reservation supposed to clear the old one,
       # so no need to disassociate manually. Also, we don't check
       # if this IP is actually an allocated EC2 elastic IP, as
       # API call will fail in that case.
-      # TODO: wrap error for non-existing elastic IP?
-      # TODO: poll instance until this IP is returned as its public IP?
-      instance.associate_elastic_ip(@ip)
-    end
 
+      errors = [AWS::EC2::Errors::IncorrectInstanceState]
+      Bosh::Common.retryable(tries: 10, sleep: 1, on: errors) do
+        instance.associate_elastic_ip(elastic_ip)
+        true # need to return true to end the retries
+      end
+    end
   end
 end
 

data/lib/cloud/aws.rb

@@ -9,23 +9,32 @@ require "httpclient"
 require "pp"
 require "set"
 require "tmpdir"
-require "uuidtools"
+require "securerandom"
 require "yajl"
 
+require "common/exec"
 require "common/thread_pool"
 require "common/thread_formatter"
 
+require "bosh/registry/client"
+
 require "cloud"
 require "cloud/aws/helpers"
 require "cloud/aws/cloud"
-require "cloud/aws/registry_client"
 require "cloud/aws/version"
 
 require "cloud/aws/aki_picker"
 require "cloud/aws/network_configurator"
 require "cloud/aws/network"
+require "cloud/aws/stemcell"
+require "cloud/aws/stemcell_creator"
 require "cloud/aws/dynamic_network"
+require "cloud/aws/manual_network"
 require "cloud/aws/vip_network"
+require "cloud/aws/instance_manager"
+require "cloud/aws/tag_manager"
+require "cloud/aws/availability_zone_selector"
+require "cloud/aws/resource_wait"
 
 module Bosh
   module Clouds

data/scripts/stemcell-copy.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+#
+# This script runs as root through sudo without the need for a password,
+# so it needs to make sure it can't be abused.
+#
+
+# make sure we have a secure PATH
+PATH=/bin:/usr/bin
+export PATH
+if [ $# -ne 2 ]; then
+  echo "usage: $0 <image-file> <block device>"
+  exit 1
+fi
+
+IMAGE="$1"
+OUTPUT="$2"
+
+# workaround for issue on 12.04 LTS, use LANG=C
+echo ${IMAGE} | LANG=C egrep '^/[A-za-z0-9_/-]+/image$' > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+  echo "ERROR: illegal image file: ${IMAGE}"
+  exit 1
+fi
+
+echo ${OUTPUT} | egrep '^/dev/[svx]+d[a-z0-9]+$' > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+  echo "ERROR: illegal device: ${OUTPUT}"
+  exit 1
+fi
+
+if [ ! -b ${OUTPUT} ]; then
+  echo "ERROR: not a device: ${OUTPUT}"
+  exit 1
+fi
+
+# copy image to block device with 1 MB block size
+tar -xzf ${IMAGE} -O root.img | dd bs=1M of=${OUTPUT}

metadata

@@ -1,96 +1,96 @@
 --- !ruby/object:Gem::Specification
 name: bosh_aws_cpi
 version: !ruby/object:Gem::Version
-  version: 0.7.0
-  prerelease: 
+  version: 1.5.0.pre.1113
+  prerelease: 6
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-18 00:00:00.000000000 Z
+date: 2013-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: 1.3.5
+        version: 1.8.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: 1.3.5
+        version: 1.8.5
 - !ruby/object:Gem::Dependency
   name: bosh_common
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 1.5.0.pre.1113
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 1.5.0.pre.1113
 - !ruby/object:Gem::Dependency
   name: bosh_cpi
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.5.1
+        version: 1.5.0.pre.1113
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.5.1
+        version: 1.5.0.pre.1113
 - !ruby/object:Gem::Dependency
-  name: httpclient
+  name: bosh-registry
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 1.5.0.pre.1113
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 1.5.0.pre.1113
 - !ruby/object:Gem::Dependency
-  name: uuidtools
+  name: httpclient
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: 2.1.2
+        version: 2.2.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: 2.1.2
+        version: 2.2.4
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -107,8 +107,10 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.8.2
-description: BOSH AWS CPI
-email: support@vmware.com
+description: ! 'BOSH AWS CPI
+
+  cfd471'
+email: support@cloudfoundry.com
 executables:
 - bosh_aws_console
 extensions: []
@@ -118,37 +120,25 @@ files:
 - lib/bosh_aws_cpi.rb
 - lib/cloud/aws.rb
 - lib/cloud/aws/aki_picker.rb
+- lib/cloud/aws/availability_zone_selector.rb
 - lib/cloud/aws/cloud.rb
 - lib/cloud/aws/dynamic_network.rb
 - lib/cloud/aws/helpers.rb
+- lib/cloud/aws/instance_manager.rb
+- lib/cloud/aws/manual_network.rb
 - lib/cloud/aws/network.rb
 - lib/cloud/aws/network_configurator.rb
-- lib/cloud/aws/registry_client.rb
+- lib/cloud/aws/resource_wait.rb
+- lib/cloud/aws/stemcell.rb
+- lib/cloud/aws/stemcell_creator.rb
+- lib/cloud/aws/tag_manager.rb
 - lib/cloud/aws/version.rb
 - lib/cloud/aws/vip_network.rb
+- scripts/stemcell-copy.sh
 - README.md
-- Rakefile
-- spec/assets/stemcell-copy
-- spec/integration/cpi_test.rb
-- spec/spec_helper.rb
-- spec/unit/aki_picker_spec.rb
-- spec/unit/attach_disk_spec.rb
-- spec/unit/cloud_spec.rb
-- spec/unit/configure_networks_spec.rb
-- spec/unit/create_disk_spec.rb
-- spec/unit/create_stemcell_spec.rb
-- spec/unit/create_vm_spec.rb
-- spec/unit/delete_disk_spec.rb
-- spec/unit/delete_stemcell_spec.rb
-- spec/unit/delete_vm_spec.rb
-- spec/unit/detach_disk_spec.rb
-- spec/unit/helpers_spec.rb
-- spec/unit/network_configurator_spec.rb
-- spec/unit/reboot_vm_spec.rb
-- spec/unit/set_vm_metadata_spec.rb
-- spec/unit/validate_deployment_spec.rb
-homepage: http://www.vmware.com
-licenses: []
+homepage: https://github.com/cloudfoundry/bosh
+licenses:
+- Apache 2.0
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -158,43 +148,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
-      segments:
-      - 0
-      hash: -1915781663242535072
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
-      segments:
-      - 0
-      hash: -1915781663242535072
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: BOSH AWS CPI
-test_files:
-- spec/assets/stemcell-copy
-- spec/integration/cpi_test.rb
-- spec/spec_helper.rb
-- spec/unit/aki_picker_spec.rb
-- spec/unit/attach_disk_spec.rb
-- spec/unit/cloud_spec.rb
-- spec/unit/configure_networks_spec.rb
-- spec/unit/create_disk_spec.rb
-- spec/unit/create_stemcell_spec.rb
-- spec/unit/create_vm_spec.rb
-- spec/unit/delete_disk_spec.rb
-- spec/unit/delete_stemcell_spec.rb
-- spec/unit/delete_vm_spec.rb
-- spec/unit/detach_disk_spec.rb
-- spec/unit/helpers_spec.rb
-- spec/unit/network_configurator_spec.rb
-- spec/unit/reboot_vm_spec.rb
-- spec/unit/set_vm_metadata_spec.rb
-- spec/unit/validate_deployment_spec.rb
-has_rdoc: 
+test_files: []

data/Rakefile

@@ -1,50 +0,0 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
-$:.unshift(File.expand_path("../../rake", __FILE__))
-
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __FILE__)
-
-require "rubygems"
-require "bundler"
-Bundler.setup(:default, :test)
-
-require "rake"
-begin
-  require "rspec/core/rake_task"
-rescue LoadError
-end
-
-require "bundler_task"
-require "ci_task"
-
-gem_helper = Bundler::GemHelper.new(Dir.pwd)
-
-desc "Build CPI gem into the pkg directory"
-task "build" do
-  gem_helper.build_gem
-end
-
-desc "Build and install CPI into system gems"
-task "install" do
-  Rake::Task["bundler:install"].invoke
-  gem_helper.install_gem
-end
-
-BundlerTask.new
-
-if defined?(RSpec)
-  namespace :spec do
-    desc "Run Unit Tests"
-    rspec_task = RSpec::Core::RakeTask.new(:unit) do |t|
-      t.pattern = "spec/unit/**/*_spec.rb"
-      t.rspec_opts = %w(--format progress --colour)
-    end
-
-    CiTask.new do |task|
-      task.rspec_task = rspec_task
-    end
-  end
-
-  desc "Run tests"
-  task :spec => %w(spec:unit)
-end

data/lib/cloud/aws/registry_client.rb

@@ -1,109 +0,0 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
-module Bosh::AwsCloud
-  class RegistryClient
-    include Helpers
-
-    attr_reader :endpoint
-    attr_reader :user
-    attr_reader :password
-
-    def initialize(endpoint, user, password)
-      @endpoint = endpoint
-
-      unless @endpoint =~ /^http:\/\//
-        @endpoint = "http://#{@endpoint}"
-      end
-
-      @user = user
-      @password = password
-
-      auth = Base64.encode64("#{@user}:#{@password}").gsub("\n", "")
-
-      @headers = {
-        "Accept" => "application/json",
-        "Authorization" => "Basic #{auth}"
-      }
-
-      @client = HTTPClient.new
-    end
-
-    ##
-    # Update instance settings in the registry
-    # @param [String] instance_id EC2 instance id
-    # @param [Hash] settings New agent settings
-    # @return [Boolean]
-    def update_settings(instance_id, settings)
-      unless settings.is_a?(Hash)
-        raise ArgumentError, "Invalid settings format, " \
-                             "Hash expected, #{settings.class} given"
-      end
-
-      payload = Yajl::Encoder.encode(settings)
-      url = "#{@endpoint}/instances/#{instance_id}/settings"
-
-      response = @client.put(url, {:body => payload, :header => @headers})
-
-      if response.status != 200
-        cloud_error("Cannot update settings for `#{instance_id}', " \
-                    "got HTTP #{response.status}")
-      end
-
-      true
-    end
-
-    ##
-    # Read instance settings from the registry
-    # @param [String] instance_id EC2 instance id
-    # @return [Hash] Agent settings
-    def read_settings(instance_id)
-      url = "#{@endpoint}/instances/#{instance_id}/settings"
-
-      response = @client.get(url, {:header => @headers})
-
-      if response.status != 200
-        cloud_error("Cannot read settings for `#{instance_id}', " \
-                    "got HTTP #{response.status}")
-      end
-
-      body = Yajl::Parser.parse(response.body)
-
-      unless body.is_a?(Hash)
-        cloud_error("Invalid registry response, Hash expected, " \
-                    "got #{body.class}: #{body}")
-      end
-
-      settings = Yajl::Parser.parse(body["settings"])
-
-      unless settings.is_a?(Hash)
-        cloud_error("Invalid settings format, " \
-                    "Hash expected, got #{settings.class}: " \
-                    "#{settings}")
-      end
-
-      settings
-
-    rescue Yajl::ParseError
-      cloud_error("Cannot parse settings for `#{instance_id}'")
-    end
-
-    ##
-    # Delete instance settings from the registry
-    # @param [String] instance_id EC2 instance id
-    # @return [Boolean]
-    def delete_settings(instance_id)
-      url = "#{@endpoint}/instances/#{instance_id}/settings"
-
-      response = @client.delete(url, {:header => @headers})
-
-      if response.status != 200
-        cloud_error("Cannot delete settings for `#{instance_id}', " \
-                    "got HTTP #{response.status}")
-      end
-
-      true
-    end
-
-  end
-
-end

data/spec/assets/stemcell-copy

@@ -1,31 +0,0 @@
-#!/bin/sh
-#
-# This script runs as root through sudo without the need for a password,
-# so it needs to make sure it can't be abused.
-#
-
-# make sure we have a secure PATH
-PATH=/bin:/usr/bin
-export PATH
-
-if [ $# -ne 1 ]; then
-  echo "usage: $0 <block device>"
-  exit 1
-fi
-
-OUTPUT="$1"
-
-# TODO perhaps this should be more restrictive?
-echo ${OUTPUT} | egrep '^/dev/[a-z0-9]+$' > /dev/null 2>&1
-if [ $? -ne 0 ]; then
-  echo "ERROR: illegal device: ${OUTPUT}"
-  exit 1
-fi
-
-if [ ! -b ${OUTPUT} ]; then
-  echo "ERROR: missing device: ${OUTPUT}"
-  exit 1
-fi
-
-# copy image to block device with 1 MB block size
-dd if=root.img if=${OUTPUT} bs=1M