bosh_aws_cpi 0.7.0 → 1.5.0.pre.1113

data/lib/cloud/aws/dynamic_network.rb

@@ -14,14 +14,8 @@ module Bosh::AwsCloud
       super
     end
 
-    ##
-    # Configures EC2 dynamic network. Right now it's a no-op,
-    # as dynamic networks are completely managed by EC2
-    # @param [AWS:EC2] ec2 instance EC2 client
-    # @param [AWS::EC2::Instance] instance EC2 instance to configure
     def configure(ec2, instance)
     end
-
   end
 end
 

data/lib/cloud/aws/helpers.rb

@@ -4,8 +4,6 @@ module Bosh::AwsCloud
 
   module Helpers
 
-    DEFAULT_TIMEOUT = 3600 # seconds
-
     ##
     # Raises CloudError exception
     #
@@ -16,72 +14,16 @@ module Bosh::AwsCloud
       raise Bosh::Clouds::CloudError, message
     end
 
-    def wait_resource(resource, target_state, state_method = :status,
-                      timeout = DEFAULT_TIMEOUT)
-
-      started_at = Time.now
-      failures = 0
-
-      # all resources but Attachment have id
-      desc = resource.respond_to?(:id) ? resource.id : resource.to_s
-
-      loop do
-        task_checkpoint
-
-        duration = Time.now - started_at
-
-        if duration > timeout
-          cloud_error("Timed out waiting for #{desc} to be #{target_state}")
-        end
-
-        if @logger
-          @logger.debug("Waiting for #{desc} to be #{target_state} " \
-                        "(#{duration}s)")
-        end
-
-        state = get_state_for(resource, state_method) do |error|
-          if block_given?
-            yield error
-          else
-            @logger.error("#{error.message}: #{desc}")
-            nil
-          end
-        end
-
-        # This is not a very strong convention, but some resources
-        # have 'error' and 'failed' states, we probably don't want to keep
-        # waiting if we're in these states. Alternatively we could introduce a
-        # set of 'loop breaker' states but that doesn't seem very helpful
-        # at the moment
-        if state == :error || state == :failed
-          cloud_error("#{desc} state is #{state}, expected #{target_state}")
-        end
-
-        break if state == target_state
-
-        sleep(1)
-      end
-
-      if @logger
-        total = Time.now - started_at
-        @logger.info("#{desc} is now #{target_state}, took #{total}s")
-      end
-    end
-
-    private
-
-    def get_state_for(resource, state_method)
-      resource.send(state_method)
-    rescue AWS::EC2::Errors::InvalidAMIID::NotFound,
-        AWS::EC2::Errors::InvalidInstanceID::NotFound,
-        AWS::Core::Resource::NotFound => e
-      # ugly workaround for AWS race conditions:
-      # 1) sometimes when we upload a stemcell and proceed to create a VM
-      #    from it, AWS reports that the AMI is missing
-      # 2) sometimes when we create a new EC2 instance, AWS reports that
-      #    the instance it returns is missing
-      # in both cases we just catch the exception, wait a little and retry...
-      yield e
+    def extract_security_group_names(networks_spec)
+      networks_spec.
+          values.
+          select { |network_spec| network_spec.has_key? "cloud_properties" }.
+          map { |network_spec| network_spec["cloud_properties"] }.
+          select { |cloud_properties| cloud_properties.has_key? "security_groups" }.
+          map { |cloud_properties| Array(cloud_properties["security_groups"]) }.
+          flatten.
+          sort.
+          uniq
     end
   end
 end

data/lib/cloud/aws/instance_manager.rb

@@ -0,0 +1,171 @@
+require "common/common"
+
+module Bosh::AwsCloud
+  class InstanceManager
+    include Helpers
+
+    attr_reader :instance
+    attr_reader :instance_params
+    attr_reader :elbs
+
+    def initialize(region, registry, az_selector=nil)
+      @region = region
+      @registry = registry
+      @logger = Bosh::Clouds::Config.logger
+      @az_selector = az_selector
+      @instance_params = {count: 1}
+    end
+
+    def create(agent_id, stemcell_id, resource_pool, networks_spec, disk_locality, environment, options)
+      @instance_params[:image_id] = stemcell_id
+      @instance_params[:instance_type] = resource_pool["instance_type"]
+      set_user_data_parameter(networks_spec)
+      set_key_name_parameter(resource_pool["key_name"], options["aws"]["default_key_name"])
+      set_security_groups_parameter(networks_spec, options["aws"]["default_security_groups"])
+      set_vpc_parameters(networks_spec)
+      set_availability_zone_parameter(
+          (disk_locality || []).map { |volume_id| @region.volumes[volume_id].availability_zone.to_s },
+          resource_pool["availability_zone"],
+          (@instance_params[:subnet].availability_zone_name if @instance_params[:subnet])
+      )
+
+      @logger.info("Creating new instance with: #{instance_params.inspect}")
+
+      # Retry the create instance operation a couple of times if we are told that the IP
+      # address is in use - it can happen when the director recreates a VM and AWS
+      # is too slow to update its state when we have released the IP address and want to
+      # realocate it again.
+      errors = [AWS::EC2::Errors::InvalidIPAddress::InUse]
+      Bosh::Common.retryable(sleep: instance_create_wait_time, tries: 10, on: errors) do |tries, error|
+        @logger.warn("IP address was in use: #{error}") if tries > 0
+        @instance = @region.instances.create(instance_params)
+      end
+
+      # We need to wait here for the instance to be running, as if we are going to
+      # attach to a load balancer, the instance must be running.
+      # If we time out, it is because the instance never gets from state running to started,
+      # so we signal the director that it is ok to retry the operation. At the moment this
+      # forever (until the operation is cancelled by the user).
+      begin
+        ResourceWait.for_instance(instance: instance, state: :running)
+      rescue Bosh::Common::RetryCountExceeded => e
+        @logger.warn("timed out waiting for #{instance.id} to be running")
+        raise Bosh::Clouds::VMCreationFailed.new(true)
+      end
+
+      @elbs = resource_pool['elbs']
+      attach_to_load_balancers if elbs
+
+      instance
+    end
+
+    def terminate(instance_id, fast=false)
+      @instance = @region.instances[instance_id]
+
+      remove_from_load_balancers
+
+      instance.terminate
+
+      @logger.info("Deleting instance settings for '#{instance.id}'")
+      @registry.delete_settings(instance.id)
+
+      if fast
+        TagManager.tag(instance, "Name", "to be deleted")
+        @logger.info("Instance #{instance_id} marked to deletion")
+        return
+      end
+
+      begin
+        @logger.info("Deleting instance '#{instance.id}'")
+        ResourceWait.for_instance(instance: instance, state: :terminated)
+      rescue AWS::EC2::Errors::InvalidInstanceID::NotFound
+        # It's OK, just means that instance has already been deleted
+      end
+    end
+
+    # Soft reboots EC2 instance
+    # @param [String] instance_id EC2 instance id
+    def reboot(instance_id)
+      instance = @region.instances[instance_id]
+
+      # There is no trackable status change for the instance being
+      # rebooted, so it's up to CPI client to keep track of agent
+      # being ready after reboot.
+      # Due to this, we can't deregister the instance from any load
+      # balancers it might be attached to, and reattach once the
+      # reboot is complete, so we just have to let the load balancers
+      # take the instance out of rotation, and put it back in once it
+      # is back up again.
+      instance.reboot
+    end
+
+    def attach_to_load_balancers
+      elb = AWS::ELB.new
+
+      elbs.each do |load_balancer|
+        lb = elb.load_balancers[load_balancer]
+        lb.instances.register(instance)
+      end
+    end
+
+    # Determines if the instance exists.
+    # @param [String] instance_id EC2 instance id
+    def has_instance?(instance_id)
+      instance = @region.instances[instance_id]
+
+      instance.exists? && instance.status != :terminated
+    end
+
+    def remove_from_load_balancers
+      elb = AWS::ELB.new
+
+      elb.load_balancers.each do |load_balancer|
+        begin
+          load_balancer.instances.deregister(instance)
+        rescue AWS::ELB::Errors::InvalidInstance
+          # ignore this, as it just means it wasn't registered
+        end
+      end
+    end
+
+    def set_key_name_parameter(resource_pool_key_name, default_aws_key_name)
+      key_name = resource_pool_key_name || default_aws_key_name
+      instance_params[:key_name] = key_name unless key_name.nil?
+    end
+
+    def set_security_groups_parameter(networks_spec, default_security_groups)
+      security_group_names = extract_security_group_names(networks_spec)
+      if security_group_names.empty?
+        instance_params[:security_groups] = default_security_groups
+      else
+        instance_params[:security_groups] = security_group_names
+      end
+    end
+
+    def set_vpc_parameters(network_spec)
+      manual_network_spec = network_spec.values.select { |spec| ["manual", nil].include? spec["type"] }.first
+      if manual_network_spec
+        instance_params[:subnet] = @region.subnets[manual_network_spec["cloud_properties"]["subnet"]]
+        instance_params[:private_ip_address] = manual_network_spec["ip"]
+      end
+    end
+
+    def set_availability_zone_parameter(volume_zones, resource_pool_zone, subnet_zone)
+      availability_zone = @az_selector.common_availability_zone(volume_zones, resource_pool_zone, subnet_zone)
+      instance_params[:availability_zone] = availability_zone if availability_zone
+    end
+
+    def set_user_data_parameter(networks_spec)
+      user_data = {registry: {endpoint: @registry.endpoint}}
+
+      spec_with_dns = networks_spec.values.select { |spec| spec.has_key? "dns" }.first
+      user_data[:dns] = {nameserver: spec_with_dns["dns"]} if spec_with_dns
+
+      @instance_params[:user_data] = Yajl::Encoder.encode(user_data)
+    end
+
+    private
+
+    def instance_create_wait_time; 30; end
+  end
+end

data/lib/cloud/aws/manual_network.rb

@@ -0,0 +1,26 @@
+module Bosh::AwsCloud
+  ##
+  #
+  class ManualNetwork < Network
+
+    attr_reader :subnet
+
+    # create manual network
+    # @param [String] name Network name
+    # @param [Hash] spec Raw network spec
+    def initialize(name, spec)
+      super
+      if @cloud_properties.nil? || !@cloud_properties.has_key?("subnet")
+        raise Bosh::Clouds::CloudError, "subnet required for manual network"
+      end
+      @subnet = @cloud_properties["subnet"]
+    end
+
+    def private_ip
+      @ip
+    end
+
+    def configure(ec2, instance)
+    end
+  end
+end

data/lib/cloud/aws/network_configurator.rb

@@ -12,11 +12,12 @@ module Bosh::AwsCloud
   class NetworkConfigurator
     include Helpers
 
+    attr_reader :vip_network, :network
+
     ##
     # Creates new network spec
     #
     # @param [Hash] spec raw network spec passed by director
-    # TODO Add network configuration examples
     def initialize(spec)
       unless spec.is_a?(Hash)
         raise ArgumentError, "Invalid spec, Hash expected, " \
@@ -24,46 +25,52 @@ module Bosh::AwsCloud
       end
 
       @logger = Bosh::Clouds::Config.logger
-      @dynamic_network = nil
+      @network = nil
       @vip_network = nil
-      @security_groups = []
 
-      spec.each_pair do |name, spec|
-        network_type = spec["type"]
+      spec.each_pair do |name, network_spec|
+        network_type = network_spec["type"] || "manual"
 
         case network_type
-        when "dynamic"
-          if @dynamic_network
-            cloud_error("More than one dynamic network for `#{name}'")
-          else
-            @dynamic_network = DynamicNetwork.new(name, spec)
-            @security_groups += extract_security_groups(spec)
-          end
-        when "vip"
-          if @vip_network
-            cloud_error("More than one vip network for `#{name}'")
+          when "dynamic"
+            cloud_error("Must have exactly one dynamic or manual network per instance") if @network
+            @network = DynamicNetwork.new(name, network_spec)
+
+          when "manual"
+            cloud_error("Must have exactly one dynamic or manual network per instance") if @network
+            @network = ManualNetwork.new(name, network_spec)
+
+          when "vip"
+            cloud_error("More than one vip network for '#{name}'") if @vip_network
+            @vip_network = VipNetwork.new(name, network_spec)
+
           else
-            @vip_network = VipNetwork.new(name, spec)
-            @security_groups += extract_security_groups(spec)
-          end
-        else
-          cloud_error("Invalid network type `#{network_type}': AWS CPI " \
-                      "can only handle `dynamic' and `vip' network types")
+            cloud_error("Invalid network type '#{network_type}' for AWS, " \
+                        "can only handle 'dynamic', 'vip', or 'manual' network types")
         end
-
       end
 
-      if @dynamic_network.nil?
-        cloud_error("At least one dynamic network should be defined")
+      unless @network
+        cloud_error("Exactly one dynamic or manual network must be defined")
       end
     end
 
+    def subnet
+      @network.subnet
+    end
+
+    def private_ip
+      vpc? ? @network.private_ip : nil
+    end
+
+    def vpc?
+      @network.is_a? ManualNetwork
+    end
+
     # Applies network configuration to the vm
     # @param [AWS:EC2] ec2 instance EC2 client
     # @param [AWS::EC2::Instance] instance EC2 instance to configure
     def configure(ec2, instance)
-      @dynamic_network.configure(ec2, instance)
-
       if @vip_network
         @vip_network.configure(ec2, instance)
       else
@@ -78,41 +85,5 @@ module Bosh::AwsCloud
         end
       end
     end
-
-    ##
-    # Returns the security groups for this network configuration, or
-    # the default security groups if the configuration does not contain
-    # security groups
-    # @param [Array] default Default security groups
-    # @return [Array] security groups
-    def security_groups(default)
-      if @security_groups.empty? && default
-        default.sort
-      else
-        @security_groups.sort
-      end
-    end
-
-    private
-
-    ##
-    # Extracts the security groups from the network configuration
-    # @param [Hash] network_spec Network specification
-    # @raise [ArgumentError] if the security groups in the network_spec
-    #   is not an Array
-    def extract_security_groups(network_spec)
-      if network_spec && network_spec["cloud_properties"]
-        cloud_properties = network_spec["cloud_properties"]
-        if cloud_properties && cloud_properties["security_groups"]
-          unless cloud_properties["security_groups"].is_a?(Array)
-            raise ArgumentError, "security groups must be an Array"
-          end
-          return cloud_properties["security_groups"]
-        end
-      end
-      []
-    end
-
   end
-
 end

data/lib/cloud/aws/resource_wait.rb

@@ -0,0 +1,189 @@
+require_relative 'helpers'
+
+module Bosh::AwsCloud
+  class ResourceWait
+    include Helpers
+
+    DEFAULT_TRIES = 12 # a sane amount of retries on AWS (~25 minutes), as things can take anywhere between a minute and forevah
+    MAX_SLEEP_EXPONENT = 8
+
+    def self.for_instance(args)
+      raise ArgumentError, "args should be a Hash, but `#{args.class}' given" unless args.is_a?(Hash)
+      instance = args.fetch(:instance) { raise ArgumentError, 'instance object required' }
+      target_state = args.fetch(:state) { raise ArgumentError, 'state symbol required' }
+      valid_states = [:running, :terminated]
+      validate_states(valid_states, target_state)
+
+      ignored_errors = [
+        AWS::EC2::Errors::InvalidInstanceID::NotFound,
+        AWS::Core::Resource::NotFound
+      ]
+
+      new.for_resource(resource: instance, errors: ignored_errors, target_state: target_state) do |current_state|
+        if target_state == :running && current_state == :terminated
+          logger.error("instance #{instance.id} terminated while starting")
+          raise Bosh::Clouds::VMCreationFailed.new(true)
+        else
+          current_state == target_state
+        end
+      end
+    end
+
+    def self.for_attachment(args)
+      attachment = args.fetch(:attachment) { raise ArgumentError, 'attachment object required' }
+      target_state = args.fetch(:state) { raise ArgumentError, 'state symbol required' }
+      valid_states = [:attached, :detached]
+      validate_states(valid_states, target_state)
+
+      ignored_errors = []
+      if target_state == :attached
+        ignored_errors << AWS::Core::Resource::NotFound
+      end
+      description = "volume %s to be %s to instance %s as device %s" % [
+          attachment.volume.id, target_state, attachment.instance.id, attachment.device
+      ]
+
+      new.for_resource(resource: attachment, errors: ignored_errors, target_state: target_state, description: description) do |current_state|
+        current_state == target_state
+      end
+    rescue AWS::Core::Resource::NotFound
+      # if an attachment is detached, AWS can reap the object and the reference is no longer found,
+      # so consider this exception a success condition if we are detaching
+      raise unless target_state == :detached
+    end
+
+    def self.for_image(args)
+      image = args.fetch(:image) { raise ArgumentError, 'image object required' }
+      target_state = args.fetch(:state) { raise ArgumentError, 'state symbol required' }
+      valid_states = [:available, :deleted]
+      validate_states(valid_states, target_state)
+
+      ignored_errors = []
+      if target_state == :available
+        ignored_errors = [AWS::EC2::Errors::InvalidAMIID::NotFound]
+      end
+
+      new.for_resource(resource: image, errors: ignored_errors, target_state: target_state, state_method: :state) do |current_state|
+        current_state == target_state
+      end
+    end
+
+    def self.for_volume(args)
+      volume = args.fetch(:volume) { raise ArgumentError, 'volume object required' }
+      target_state = args.fetch(:state) { raise ArgumentError, 'state symbol required' }
+      valid_states = [:available, :deleted]
+      validate_states(valid_states, target_state)
+
+      new.for_resource(resource: volume, target_state: target_state) do |current_state|
+        current_state == target_state
+      end
+    rescue AWS::EC2::Errors::InvalidVolume::NotFound
+      # if an volume is deleted, AWS can reap the object and the reference is no longer found,
+      # so consider this exception a success condition if we are deleting
+      raise unless target_state == :deleted
+    end
+
+    def self.for_snapshot(args)
+      snapshot = args.fetch(:snapshot) { raise ArgumentError, 'snapshot object required' }
+      target_state = args.fetch(:state) { raise ArgumentError, 'state symbol required' }
+      valid_states = [:completed]
+      validate_states(valid_states, target_state)
+
+      new.for_resource(resource: snapshot, target_state: target_state, tries: 18) do |current_state|
+        current_state == target_state
+      end
+    end
+
+    def self.for_subnet(args)
+      subnet = args.fetch(:subnet) { raise ArgumentError, 'subnet object required' }
+      target_state = args.fetch(:state) { raise ArgumentError, 'state symbol required' }
+      valid_states = [:available]
+      validate_states(valid_states, target_state)
+
+      ignored_errors = [AWS::EC2::Errors::InvalidSubnetID::NotFound]
+
+      new.for_resource(resource: subnet, target_state: target_state, errors: ignored_errors, state_method: :state) do |current_state|
+        current_state == target_state
+      end
+    end
+
+    def self.for_sgroup(args)
+      sgroup = args.fetch(:sgroup) { raise ArgumentError, 'sgroup object required' }
+      target_state = args.fetch(:state) { raise ArgumentError, 'state symbol required' }
+      valid_states = [true, false]
+      validate_states(valid_states, target_state)
+
+      new.for_resource(resource: sgroup, target_state: true, state_method: :exists?) do |current_state|
+        current_state == target_state
+      end
+    end
+
+    def self.validate_states(valid_states, target_state)
+      unless valid_states.include?(target_state)
+        raise ArgumentError, "target state must be one of #{valid_states.join(', ')}, `#{target_state}' given"
+      end
+    end
+
+    def self.logger
+      Bosh::Clouds::Config.logger
+    end
+
+    def self.task_checkpoint
+      Bosh::Clouds::Config.task_checkpoint
+    end
+
+    def initialize
+      @started_at = Time.now
+    end
+
+    def for_resource(args)
+      resource = args.fetch(:resource)
+      state_method = args.fetch(:state_method, :status)
+      errors = args.fetch(:errors, [])
+      desc = args.fetch(:description) { resource.id }
+      tries = args.fetch(:tries, DEFAULT_TRIES).to_i
+      target_state = args.fetch(:target_state)
+
+      sleep_cb = self.class.sleep_callback("Waiting for #{desc} to be #{target_state}", tries)
+      errors << AWS::EC2::Errors::RequestLimitExceeded
+      ensure_cb = Proc.new do |retries|
+        cloud_error("Timed out waiting for #{desc} to be #{target_state}, took #{time_passed}s") if retries == tries
+      end
+
+      state = nil
+      Bosh::Common.retryable(tries: tries, sleep: sleep_cb, on: errors, ensure: ensure_cb ) do
+        Bosh::AwsCloud::ResourceWait.task_checkpoint
+
+        state = resource.method(state_method).call
+
+        if state == :error || state == :failed
+          raise Bosh::Clouds::CloudError, "#{desc} state is #{state}, expected #{target_state}, took #{time_passed}s"
+        end
+
+        # the yielded block should return true if we have reached the target state
+        yield state
+      end
+
+      Bosh::AwsCloud::ResourceWait.logger.info("#{desc} is now #{state}, took #{time_passed}s")
+    rescue Bosh::Common::RetryCountExceeded => e
+      Bosh::AwsCloud::ResourceWait.logger.error("Timed out waiting for #{desc} state is #{state}, expected to be #{target_state}, took #{time_passed}s")
+      raise e
+    end
+
+    def time_passed
+      Time.now - @started_at
+    end
+
+    private
+
+    def self.sleep_callback(description, tries)
+      lambda do |num_tries, error|
+        sleep_time = 2**[num_tries, MAX_SLEEP_EXPONENT].min # Exp backoff: 1, 2, 4, 8 ... up to max 256
+        Bosh::AwsCloud::ResourceWait.logger.debug("#{error.class}: `#{error.message}'") if error
+        Bosh::AwsCloud::ResourceWait.logger.debug("#{description}, retrying in #{sleep_time} seconds (#{num_tries}/#{tries})")
+        sleep_time
+      end
+    end
+  end
+end
+

data/lib/cloud/aws/stemcell.rb

@@ -0,0 +1,68 @@
+module Bosh::AwsCloud
+  class Stemcell
+    include Helpers
+
+    attr_reader :ami, :snapshots
+
+    def self.find(region, id)
+      image = region.images[id]
+      raise Bosh::Clouds::CloudError, "could not find AMI '#{id}'" unless image.exists?
+      new(region, image)
+    end
+
+    def initialize(region, image)
+      @region = region
+      @ami = image
+      @snapshots = []
+    end
+
+    def delete
+      memoize_snapshots
+
+      ami.deregister
+
+      # Wait for the AMI to be deregistered, or the snapshot deletion will fail,
+      # as the AMI is still in use.
+      ResourceWait.for_image(image: ami, state: :deleted)
+
+      delete_snapshots
+      logger.info("deleted stemcell '#{id}'")
+    rescue AWS::EC2::Errors::AuthFailure => e
+      # If we get an auth failure from the deregister call, it means we don't own the AMI
+      # and we were just faking it, so we can just return pretending that we deleted it.
+      logger.info("deleted fake stemcell '#{id}")
+    end
+
+    def id
+      ami.id
+    end
+
+    def root_device_name
+      ami.root_device_name
+    end
+
+    def memoize_snapshots
+      # .to_h is used as the AWS API documentation isn't trustworthy:
+      # it says block_device_mappings retruns a Hash, but in reality it flattens it!
+      ami.block_device_mappings.to_h.each do |device, map|
+        snapshot_id = map[:snapshot_id]
+        if id
+          logger.debug("queuing snapshot '#{snapshot_id}' for deletion")
+          snapshots << snapshot_id
+        end
+      end
+    end
+
+    def delete_snapshots
+      snapshots.each do |id|
+        logger.info("cleaning up snapshot '#{id}'")
+        snapshot = @region.snapshots[id]
+        snapshot.delete
+      end
+    end
+
+    def logger
+      Bosh::Clouds::Config.logger
+    end
+  end
+end