chef-provisioning-aws 1.0.4 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd5c21364731d0404bcc87495c3a1e7fddc9b107
-  data.tar.gz: b0f57238cd577c5be3b7ee652079d1185a3fef8f
+  metadata.gz: 66de7d20ac16f1c99866f9d4195f38586bfb667d
+  data.tar.gz: 5cbae8b8fcc08af3538333bbf6bc29c96beb0f22
 SHA512:
-  metadata.gz: 9676f4c941de6c694f9df2d58d04b0fd03a5958a23c10cacae91a50c4bfe52d1aadf249e36346ab2c003f4931d127d599a744ba9740505ee2d3c457f08ba6d79
-  data.tar.gz: c84c5149b0127cfabed5ad71f967a48cc0dce1cf05ebc80b8b90e3433b1ac485f5ff2a42459d4167aa1e954112617cf8cf8d4f295c6623b547ef97f8c60c85f3
+  metadata.gz: 36907c00f23caf788569f528f6b12ad64afb4c316f02794dd05927c06e234b7271257d7d83c85f517c7bb370928507fe2616bb9379e5b161e48cdce50dc3fed6
+  data.tar.gz: 616a2bcd0b104326714aa001462f042eedec30d06825b5e9556b6e326a03037833ddd4d79879183bd4b20a7335272140f253be91b380817acf4a2b13c4c298ae

data/README.md

@@ -12,3 +12,21 @@ An implementation of the AWS driver using the AWS Ruby SDK (v1).  It also implem
 * Autoscaling Groups
 * SSH Key pairs
 * Launch configs
+
+# Running Integration Tests
+
+To run the integration tests execute `bundle exec rake integration`.  If you have not set it up,
+you should see an error message about a missing environment variable `AWS_TEST_DRIVER`.  You can add
+this as a normal environment variable or set it for a single run with `AWS_TEST_DRIVER=aws::eu-west-1
+bundle exec rake integration`.  The format should match what `with_driver` expects.
+
+You will also need to have configured your `~/.aws/config` or environment variables with your
+AWS credentials.
+
+This creates real objects within AWS.  The tests make their best effort to delete these objects
+after each test finishes but errors can happen which prevent this.  Be aware that this may charge
+you!
+
+If you find the tests leaving behind resources during normal conditions (IE, not when there is an
+unexpected exception) please file a bug.  Most objects can be cleaned up by deleting the `test_vpc`
+from within the AWS browser console.

data/Rakefile

@@ -10,3 +10,8 @@ RSpec::Core::RakeTask.new(:spec) do |spec|
   # TODO add back integration tests whenever we have strategy for keys
   spec.exclude_pattern = 'spec/integration/**/*_spec.rb'
 end
+
+desc "Run Integration Specs"
+RSpec::Core::RakeTask.new(:integration) do |spec|
+  spec.pattern = 'spec/integration/**/*_spec.rb'
+end

data/lib/chef/provider/aws_ebs_volume.rb

@@ -47,8 +47,8 @@ class Chef::Provider::AwsEbsVolume < Chef::Provisioning::AWSDriver::AWSProvider
 
   def update_aws_object(volume)
     if initial_options.has_key?(:availability_zone)
-      if initial_options[:availability_zone] != volume.availability_zone_name
-        raise "#{new_resource}.availability_zone is #{new_resource.availability_zone}, but actual volume has availability_zone_name set to #{volume.availability_zone_name}.  Cannot be modified!"
+      if availability_zone != volume.availability_zone_name
+        raise "#{new_resource}.availability_zone is #{availability_zone}, but actual volume has availability_zone_name set to #{volume.availability_zone_name}.  Cannot be modified!"
       end
     end
     if initial_options.has_key?(:size)
@@ -99,7 +99,7 @@ class Chef::Provider::AwsEbsVolume < Chef::Provisioning::AWSDriver::AWSProvider
   def initial_options
     @initial_options ||= begin
       options = {}
-      options[:availability_zone] = new_resource.availability_zone if !new_resource.availability_zone.nil?
+      options[:availability_zone] = availability_zone              if !new_resource.availability_zone.nil?
       options[:size]              = new_resource.size              if !new_resource.size.nil?
       options[:snapshot_id]       = new_resource.snapshot          if !new_resource.snapshot.nil?
       options[:iops]              = new_resource.iops              if !new_resource.iops.nil?
@@ -146,7 +146,7 @@ class Chef::Provider::AwsEbsVolume < Chef::Provisioning::AWSDriver::AWSProvider
     end
     volume
   end
-    
+
   def wait_for_volume_status(volume, expected_status)
     initial_status = volume.status
     log_callback = proc {
@@ -204,4 +204,14 @@ class Chef::Provider::AwsEbsVolume < Chef::Provisioning::AWSDriver::AWSProvider
       volume
     end
   end
+
+  def availability_zone
+    az = new_resource.availability_zone
+    if /^#{region}/ =~ az
+      Chef::Log.warn("availability_zone attribute should only be set to the letter designation. Attempting to use '#{az[-1]}' to correct the issue.")
+    elsif az.length == 1
+      az = "#{region}#{az[-1]}"
+    end
+    az
+  end
 end

data/lib/chef/provider/aws_image.rb

@@ -0,0 +1,31 @@
+require 'chef/provisioning/aws_driver/aws_provider'
+
+class Chef::Provider::AwsImage < Chef::Provisioning::AWSDriver::AWSProvider
+  def destroy_aws_object(image)
+    instance_id = image.tags['From-Instance']
+    Chef::Log.debug("Found From-Instance tag [#{instance_id}] on #{image.id}")
+    unless instance_id
+      # This is an old image and doesn't have the tag added - lets try and find it from the block device mapping
+      image.block_device_mappings.map do |dev, opts|
+        snapshot = ec2.snapshots[opts[:snapshot_id]]
+        desc = snapshot.description
+        m = /CreateImage\(([^\)]+)\)/.match(desc)
+        if m
+          Chef::Log.debug("Found [#{instance_id}] from snapshot #{snapshot.id} on #{image.id}")
+          instance_id = m[1]
+        end
+      end
+    end
+    converge_by "delete image #{new_resource} in #{region}" do
+      image.delete
+    end
+    if instance_id
+      # As part of the image creation process, the source instance was automatically
+      # destroyed - we just need to make sure that has completed successfully
+      instance = new_resource.driver.ec2.instances[instance_id]
+      converge_by "waiting until instance #{instance.id} is :terminated" do
+        wait_for_status(instance, :terminated, [AWS::EC2::Errors::InvalidInstanceID::NotFound])
+      end
+    end
+  end
+end

data/lib/chef/provider/aws_instance.rb

@@ -0,0 +1,14 @@
+require 'chef/provisioning/aws_driver/aws_provider'
+
+class Chef::Provider::AwsInstance < Chef::Provisioning::AWSDriver::AWSProvider
+  def destroy_aws_object(instance)
+    converge_by "delete instance #{new_resource} in VPC #{instance.vpc.id} in #{region}" do
+      instance.delete
+    end
+    converge_by "waited until instance #{new_resource} is :terminated" do
+      # When purging, we must wait until the instance is fully terminated - thats the only way
+      # to delete the network interface that I can see
+      wait_for_status(instance, :terminated, [AWS::EC2::Errors::InvalidInstanceID::NotFound])
+    end
+  end
+end

data/lib/chef/provider/aws_load_balancer.rb

@@ -0,0 +1,9 @@
+require 'chef/provisioning/aws_driver/aws_provider'
+
+class Chef::Provider::AwsLoadBalancer < Chef::Provisioning::AWSDriver::AWSProvider
+  def destroy_aws_object(load_balancer)
+    converge_by "delete load balancer #{new_resource.name} (#{load_balancer.id}) in VPC #{load_balancer.vpc.id} in #{region}" do
+      load_balancer.delete
+    end
+  end
+end

data/lib/chef/provider/aws_network_interface.rb

@@ -0,0 +1,209 @@
+require 'chef/provisioning/aws_driver/aws_provider'
+require 'cheffish'
+require 'date'
+require 'retryable'
+
+class Chef::Provider::AwsNetworkInterface < Chef::Provisioning::AWSDriver::AWSProvider
+  class NetworkInterfaceStatusTimeoutError < TimeoutError
+    def initialize(new_resource, initial_status, expected_status)
+      super("timed out waiting for #{new_resource} status to change from #{initial_status} to #{expected_status}!")
+    end
+  end
+
+  class NetworkInterfaceStatusTimeoutError < TimeoutError
+    def initialize(new_resource, initial_status, expected_status)
+      super("timed out waiting for #{new_resource} status to change from #{initial_status} to #{expected_status}!")
+    end
+  end
+
+  class NetworkInterfaceInvalidStatusError < RuntimeError
+    def initialize(new_resource, status)
+      super("#{new_resource} is in #{status} state!")
+    end
+  end
+
+  def action_create
+    eni = super
+
+    if !new_resource.machine.nil?
+      update_eni(eni)
+    end
+  end
+
+  protected
+
+  def create_aws_object
+    eni = nil
+    converge_by "create new #{new_resource} in #{region}" do
+      eni = new_resource.driver.ec2.network_interfaces.create(options)
+      eni.tags['Name'] = new_resource.name
+    end
+
+    converge_by "wait for new #{new_resource} in #{region} to become available" do
+      wait_for_eni_status(eni, :available)
+      eni
+    end
+  end
+
+  def update_aws_object(eni)
+    if options.has_key?(:subnet)
+      if Chef::Resource::AwsSubnet.get_aws_object(options[:subnet], resource: new_resource) != eni.subnet
+        raise "#{new_resource} subnet is #{new_resource.subnet}, but actual network interface has subnet set to #{eni.subnet_id}.  Cannot be modified!"
+      end
+    end
+
+    # TODO implement private ip reassignment
+    if options.has_key?(:private_ip_address)
+      if options[:private_ip_address] != eni.private_ip_address
+        raise "#{new_resource} private IP is #{new_resource.private_ip_address}, but actual network interface has private IP set to #{eni.private_ip_address}.  Private IP reassignment not implemented. Cannot be modified!"
+      end
+    end
+
+    if options.has_key?(:description)
+      if options[:description] != eni.description
+        converge_by "set #{new_resource} description to #{new_resource.description}" do
+          eni.client.modify_network_interface_attribute(:network_interface_id => eni.network_interface_id,
+                                                        :description => {
+                                                            :value => new_resource.description
+                                                        })
+        end
+      end
+    end
+
+    if options.has_key?(:security_groups)
+      groups = new_resource.security_groups.map { |sg|
+        Chef::Resource::AwsSecurityGroup.get_aws_object(sg, resource: new_resource)
+      }
+      if groups.sort != eni.security_groups.sort
+        converge_by "set #{new_resource} security groups to #{groups}" do
+          eni.set_security_groups(groups)
+          eni
+        end
+      end
+    end
+
+    eni
+  end
+
+  def destroy_aws_object(eni)
+    detach(eni) if eni.status == :in_use
+    delete(eni)
+  end
+
+  private
+
+  def expected_instance
+    # use instance if already set
+    @expected_instance ||= new_resource.machine ?
+      # if not, and machine is set, find and return the instance
+        Chef::Resource::AwsInstance.get_aws_object(new_resource.machine, resource: new_resource) :
+      # otherwise return nil
+        nil
+  end
+
+  def options
+    @options ||= begin
+      options = {}
+      options[:subnet] = new_resource.subnet if !new_resource.subnet.nil?
+      options[:private_ip_address] = new_resource.private_ip_address if !new_resource.private_ip_address.nil?
+      options[:description] = new_resource.description if !new_resource.description.nil?
+      options[:security_groups] = new_resource.security_groups if !new_resource.security_groups.nil?
+      options[:device_index] = new_resource.device_index if !new_resource.device_index.nil?
+
+      AWSResource.lookup_options(options, resource: new_resource)
+    end
+  end
+
+  def update_eni(eni)
+    status = eni.status
+    #
+    # If we were told to attach the network interface to a machine, do so
+    #
+    if expected_instance.is_a?(AWS::EC2::Instance)
+      case status
+      when :available
+        attach(eni)
+      when :in_use
+        # We don't want to attempt to reattach to the same instance or device index
+        attachment = current_attachment(eni)
+        if attachment.instance != expected_instance || (options[:device_index] && attachment.device_index != new_resource.device_index)
+          detach(eni)
+          attach(eni)
+        end
+      when nil
+        raise NetworkInterfaceNotFoundError.new(new_resource)
+      else
+        raise NetworkInterfaceInvalidStatusError.new(new_resource, status)
+      end
+
+    #
+    # If we were told to set the machine to false, detach it.
+    #
+    else
+      case status
+      when nil
+        Chef::Log.warn NetworkInterfaceNotFoundError.new(new_resource)
+      when :in_use
+        detach(eni)
+      end
+    end
+    eni
+  end
+    
+  def wait_for_eni_status(eni, expected_status)
+    initial_status = eni.status
+    log_callback = proc {
+      Chef::Log.info("waiting for #{new_resource} status to change to #{expected_status}...")
+    }
+
+    Retryable.retryable(:tries => 30, :sleep => 2, :on => NetworkInterfaceStatusTimeoutError, :ensure => log_callback) do
+      raise NetworkInterfaceStatusTimeoutError.new(new_resource, initial_status, expected_status) if eni.status != expected_status
+    end
+  end
+
+  def detach(eni)
+    attachment   = current_attachment(eni)
+    instance     = attachment.instance
+
+    converge_by "detach #{new_resource} from #{instance.instance_id}" do
+      eni.detach
+    end
+
+    converge_by "wait for #{new_resource} to detach" do
+      wait_for_eni_status(eni, :available)
+      eni
+    end
+  end
+
+  def attach(eni)
+    converge_by "attach #{new_resource} to #{new_resource.machine} (#{expected_instance.instance_id})" do
+      eni.attach(expected_instance, options)
+    end
+
+    converge_by "wait for #{new_resource} to attach" do
+      wait_for_eni_status(eni, :in_use)
+      eni
+    end
+  end
+
+  def current_attachment(eni)
+    eni.attachment
+  end
+
+  def delete(eni)
+    converge_by "delete #{new_resource} in #{region}" do
+      eni.delete
+    end
+
+    converge_by "wait for #{new_resource} in #{region} to delete" do
+      log_callback = proc {
+        Chef::Log.info('waiting for network interface to delete...')
+      }
+
+      Retryable.retryable(:tries => 30, :sleep => 2, :on => NetworkInterfaceStatusTimeoutError, :ensure => log_callback) do
+        raise NetworkInterfaceStatusTimeoutError.new(new_resource, 'exists', 'deleted') if eni.exists?
+      end
+      eni
+    end
+  end
+end

data/lib/chef/provider/aws_security_group.rb

@@ -71,7 +71,7 @@ class Chef::Provider::AwsSecurityGroup < Chef::Provisioning::AWSDriver::AWSProvi
     when Array
       # [ { port: X, protocol: Y, sources: [ ... ]}]
       new_resource.inbound_rules.each do |rule|
-        port_ranges = get_port_ranges(port_range: rule[:port], protocol: rule[:protocol])
+        port_ranges = get_port_ranges(rule)
         add_rule(desired_rules, port_ranges, get_actors(vpc, rule[:sources]))
       end
 
@@ -115,7 +115,7 @@ class Chef::Provider::AwsSecurityGroup < Chef::Provisioning::AWSDriver::AWSProvi
     when Array
       # [ { port: X, protocol: Y, sources: [ ... ]}]
       new_resource.outbound_rules.each do |rule|
-        port_ranges = get_port_ranges(port_range: rule[:port], protocol: rule[:protocol])
+        port_ranges = get_port_ranges(rule)
         add_rule(desired_rules, port_ranges, get_actors(vpc, rule[:destinations]))
       end
 
@@ -204,11 +204,16 @@ class Chef::Provider::AwsSecurityGroup < Chef::Provisioning::AWSDriver::AWSProvi
       [ { port_range: port_spec, protocol: :tcp } ]
     when Array
       port_spec.map { |p| get_port_ranges(p) }.flatten
+    when :icmp
+      { port_range: port_spec, protocol: :icmp }
     when Hash
+      port_range = port_spec[:port_range] || port_spec[:ports] || port_spec[:port]
+      port_range = port_range..port_range if port_range.is_a?(Integer)
       if port_spec[:protocol]
-        [ { port_range: port_spec[:port_range] || port_spec[:port], protocol: port_spec[:protocol].to_s.to_sym } ]
+        port_range ||= -1..-1
+        [ { port_range: port_range, protocol: port_spec[:protocol].to_s.to_sym || :tcp } ]
       else
-        get_port_ranges(port_spec[:port_range] || port_spec[:port])
+        get_port_ranges(port_range)
       end
       # The to_s.to_sym dance is because if you specify a protocol number, AWS symbolifies it,
       # but 26.to_sym doesn't work (so we have to to_s it first).

data/lib/chef/provider/aws_subnet.rb

@@ -51,8 +51,23 @@ class Chef::Provider::AwsSubnet < Chef::Provisioning::AWSDriver::AWSProvider
   end
 
   def destroy_aws_object(subnet)
+    if purging
+      # TODO possibly convert to http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/EC2/Client.html#terminate_instances-instance_method
+      p = Chef::ChefFS::Parallelizer.new(5)
+      p.parallel_do(subnet.instances.to_a) do |instance|
+        Cheffish.inline_resource(self, action) do
+          aws_instance instance do
+            action :purge
+          end
+        end
+      end
+    end
     converge_by "delete subnet #{new_resource.name} in VPC #{new_resource.vpc} in #{region}" do
-      subnet.delete
+      # If the subnet doesn't exist we can't check state on it - state can only be :pending or :available
+      begin
+        subnet.delete
+      rescue AWS::EC2::Errors::InvalidSubnetID::NotFound
+      end
     end
   end
 

data/lib/chef/provider/aws_vpc.rb

@@ -55,6 +55,22 @@ class Chef::Provider::AwsVpc < Chef::Provisioning::AWSDriver::AWSProvider
   end
 
   def destroy_aws_object(vpc)
+    if purging
+      vpc.subnets.each do |s|
+        Cheffish.inline_resource(self, action) do # if action isn't defined, we want :purge
+          aws_subnet s do
+            action :purge
+          end
+        end
+      end
+      # If any of the below resources start needing complicated delete logic (dependent resources needing to
+      # be deleted) move that logic into `delete_aws_resource` and add the purging logic to the resource
+      vpc.network_acls.each       { |o| o.delete unless o.default? }
+      vpc.network_interfaces.each { |o| o.delete }
+      vpc.route_tables.each       { |o| o.delete unless o.main? }
+      vpc.security_groups.each    { |o| o.delete unless o.name == 'default' }
+    end
+
     # Detach or destroy the internet gateway
     ig = vpc.internet_gateway
     if ig

data/lib/chef/provisioning/aws_driver/aws_provider.rb

@@ -3,6 +3,7 @@ require 'chef/provisioning/aws_driver/aws_resource'
 require 'chef/provisioning/aws_driver/aws_resource_with_entry'
 require 'chef/provisioning/chef_managed_entry_store'
 require 'chef/provisioning/chef_provider_action_handler'
+require 'retryable'
 
 module Chef::Provisioning::AWSDriver
 class AWSProvider < Chef::Provider::LWRPBase
@@ -10,6 +11,12 @@ class AWSProvider < Chef::Provider::LWRPBase
 
   AWSResource = Chef::Provisioning::AWSDriver::AWSResource
 
+  class StatusTimeoutError < TimeoutError
+    def initialize(aws_object, initial_status, expected_status)
+      super("timed out waiting for #{aws_object.id} status to change from #{initial_status.inspect} to #{expected_status.inspect}!")
+    end
+  end
+
   def action_handler
     @action_handler ||= Chef::Provisioning::ChefProviderActionHandler.new(self)
   end
@@ -124,6 +131,18 @@ class AWSProvider < Chef::Provider::LWRPBase
     aws_object
   end
 
+  # TODO having a @purging flag feels weird
+  action :purge do
+    @purging = true
+    begin
+      action_destroy
+    ensure
+      @purging = false
+    end
+  end
+
+  attr_reader :purging
+
   action :destroy do
     desired_driver = new_resource.driver
     desired_id = new_resource.public_send(new_resource.class.aws_id_attribute) if new_resource.class.aws_id_attribute
@@ -202,5 +221,30 @@ class AWSProvider < Chef::Provider::LWRPBase
     raise NotImplementedError, :destroy_aws_object
   end
 
+  # Wait until aws_object obtains one of expected_status
+  #
+  # @param aws_object Aws SDK Object to check status on
+  # @param expected_status [Symbol,Array<Symbol>] Final status(s) to look for
+  # @param acceptable_errors [Exception,Array<Exception>] Acceptable errors that are caught and squelched
+  # @param tries [Integer] Number of times to check status
+  # @param sleep [Integer] Time to wait between checking status
+  #
+  def wait_for_status(aws_object, expected_status, acceptable_errors = [], tries=60, sleep=5)
+    acceptable_errors = [acceptable_errors].flatten
+    expected_status = [expected_status].flatten
+    current_status = aws_object.status
+
+    Retryable.retryable(:tries => tries, :sleep => sleep, :on => StatusTimeoutError) do |retries, exception|
+      action_handler.report_progress "waited #{retries*sleep}/#{tries*sleep}s for #{aws_object.id} status to change to #{expected_status.inspect}..."
+      begin
+        current_status = aws_object.status
+        unless expected_status.include?(current_status)
+          raise StatusTimeoutError.new(aws_object, current_status, expected_status)
+        end
+      rescue *acceptable_errors
+      end
+    end
+  end
+
 end
 end