chef-provisioning-aws 1.1.1 → 1.2.0

data/lib/chef/provisioning/aws_driver/aws_provider.rb

@@ -121,6 +121,8 @@ class AWSProvider < Chef::Provider::LWRPBase
       aws_object = create_aws_object
     end
 
+    converge_tags(aws_object)
+
     #
     # Associate the managed entry with the AWS object
     #
@@ -221,6 +223,45 @@ class AWSProvider < Chef::Provider::LWRPBase
     raise NotImplementedError, :destroy_aws_object
   end
 
+  # Update AWS resource tags
+  #
+  # AWS resources which include the TaggedItem Module
+  # will have an 'aws_tags' attribute available.
+  # The 'aws_tags' Hash will apply all the tags within
+  # the hash, and remove existing tags not included within
+  # the hash.  The 'Name' tag will not removed.  The 'Name'
+  # tag can still be updated in the hash.
+  #
+  # @param aws_object Aws SDK Object to update tags
+  #
+  def converge_tags(aws_object)
+    desired_tags = new_resource.aws_tags
+    # If aws_tags were not provided we exit
+    if desired_tags.nil?
+      Chef::Log.debug "aws_tags not provided, nothing to converge"
+      return
+    end
+    current_tags = aws_object.tags.to_h
+    # AWS always returns tags as strings, and we don't want to overwrite a
+    # tag-as-string with the same tag-as-symbol
+    desired_tags = Hash[desired_tags.map {|k, v| [k.to_s, v.to_s] }]
+    tags_to_update = desired_tags.reject {|k,v| current_tags[k] == v}
+    tags_to_delete = current_tags.keys - desired_tags.keys
+    # We don't want to delete `Name`, just all other tags
+    tags_to_delete.delete('Name')
+
+    unless tags_to_update.empty?
+      converge_by "applying tags #{tags_to_update}" do
+        aws_object.tags.set(tags_to_update)
+      end
+    end
+    unless tags_to_delete.empty?
+      converge_by "deleting tags #{tags_to_delete.inspect}" do
+        aws_object.tags.delete(*tags_to_delete)
+      end
+    end
+  end
+
   # Wait until aws_object obtains one of expected_status
   #
   # @param aws_object Aws SDK Object to check status on

data/lib/chef/provisioning/aws_driver/aws_resource_with_entry.rb

@@ -3,6 +3,13 @@ require 'chef/provisioning/aws_driver/resources'
 
 # Common AWS resource - contains metadata that all AWS resources will need
 class Chef::Provisioning::AWSDriver::AWSResourceWithEntry < Chef::Provisioning::AWSDriver::AWSResource
+
+  # This should be a hash of tags to apply to the AWS object
+  #
+  # @param aws_tags [Hash] Should be a hash of keys & values to add.  Keys and values
+  #        can be provided as symbols or strings, but will be stored in AWS as strings.
+  attribute :aws_tags, kind_of: Hash
+
   #
   # Dissociate the ID of this object from Chef.
   #

data/lib/chef/provisioning/aws_driver/driver.rb

@@ -20,6 +20,8 @@ require 'chef/provisioning/aws_driver/credentials'
 
 require 'yaml'
 require 'aws-sdk-v1'
+require 'retryable'
+
 
 # loads the entire aws-sdk
 AWS.eager_autoload!
@@ -84,10 +86,14 @@ module AWSDriver
         updates << "  attach subnets #{lb_options[:subnets].join(', ')}" if lb_options[:subnets]
         updates << "  with listeners #{lb_options[:listeners]}" if lb_options[:listeners]
         updates << "  with security groups #{lb_options[:security_groups]}" if lb_options[:security_groups]
+        updates << "  with tags #{lb_options[:aws_tags]}" if lb_options[:aws_tags]
 
 
+        lb_aws_tags = lb_options[:aws_tags]
+        lb_options.delete(:aws_tags)
         action_handler.perform_action updates do
           actual_elb = elb.load_balancers.create(lb_spec.name, lb_options)
+          lb_options[:aws_tags] = lb_aws_tags
 
           lb_spec.reference = {
             'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
@@ -269,6 +275,35 @@ module AWSDriver
         end
       end
 
+      # GRRRR curse you AWS and your crappy tagging support for ELBs
+      read_tags_block = lambda {|aws_object|
+        resp = elb.client.describe_tags load_balancer_names: [aws_object.name]
+        tags = {}
+        resp.data[:tag_descriptions] && resp.data[:tag_descriptions].each do |td|
+          td[:tags].each do |t|
+            tags[t[:key]] = t[:value]
+          end
+        end
+        tags
+      }
+
+      set_tags_block = lambda {|aws_object, desired_tags|
+        aws_form_tags = []
+        desired_tags.each do |k, v|
+          aws_form_tags << {key: k, value: v}
+        end
+        elb.client.add_tags load_balancer_names: [aws_object.name], tags: aws_form_tags
+      }
+
+      delete_tags_block=lambda {|aws_object, tags_to_delete|
+        aws_form_tags = []
+        tags_to_delete.each do |k, v|
+          aws_form_tags << {key: k}
+        end
+        elb.client.remove_tags load_balancer_names: [aws_object.name], tags: aws_form_tags
+      }
+      converge_tags(actual_elb, lb_options[:aws_tags], action_handler, read_tags_block, set_tags_block, delete_tags_block)
+
       # Update instance list, but only if there are machines specified
       if machine_specs
         actual_instance_ids = actual_elb.instances.map { |i| i.instance_id }
@@ -326,22 +361,24 @@ module AWSDriver
     # Image methods
     def allocate_image(action_handler, image_spec, image_options, machine_spec, machine_options)
       actual_image = image_for(image_spec)
+      aws_tags = image_options.delete(:aws_tags) || {}
       if actual_image.nil? || !actual_image.exists? || actual_image.state == :failed
         action_handler.perform_action "Create image #{image_spec.name} from machine #{machine_spec.name} with options #{image_options.inspect}" do
           image_options[:name] ||= image_spec.name
           image_options[:instance_id] ||= machine_spec.reference['instance_id']
           image_options[:description] ||= "Image #{image_spec.name} created from machine #{machine_spec.name}"
           Chef::Log.debug "AWS Image options: #{image_options.inspect}"
-          image = ec2.images.create(image_options.to_hash)
-          image.add_tag('From-Instance', :value => image_options[:instance_id]) if image_options[:instance_id]
+          actual_image = ec2.images.create(image_options.to_hash)
           image_spec.reference = {
             'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
-            'image_id' => image.id,
+            'image_id' => actual_image.id,
             'allocated_at' => Time.now.to_i
           }
           image_spec.driver_url = driver_url
         end
       end
+      aws_tags['From-Instance'] = image_options[:instance_id] if image_options[:instance_id]
+      converge_tags(actual_image, aws_tags, action_handler)
     end
 
     def ready_image(action_handler, image_spec, image_options)
@@ -359,22 +396,12 @@ module AWSDriver
     end
 
     def destroy_image(action_handler, image_spec, image_options)
-      actual_image = image_for(image_spec)
-      if actual_image.nil? || !actual_image.exists?
-        Chef::Log.warn "Image #{image_spec.name} doesn't exist"
-      else
-        snapshots = actual_image.block_device_mappings.map do |dev, opts|
-          ec2.snapshots[opts[:snapshot_id]]
-        end
-        action_handler.perform_action "De-registering image #{image_spec.name}" do
-          actual_image.deregister
-        end
-        if snapshots.any?
-          action_handler.perform_action "Deleting image #{image_spec.name} snapshots" do
-            snapshots.each do |snap|
-              snap.delete
-            end
-          end
+      # TODO the driver should automatically be set by `inline_resource`
+      d = self
+      Provisioning.inline_resource(action_handler) do
+        aws_image image_spec.name do
+          action :destroy
+          driver d
         end
       end
     end
@@ -402,25 +429,28 @@ EOD
     # Machine methods
     def allocate_machine(action_handler, machine_spec, machine_options)
       actual_instance = instance_for(machine_spec)
+      bootstrap_options = bootstrap_options_for(action_handler, machine_spec, machine_options)
+
       if actual_instance == nil || !actual_instance.exists? || actual_instance.status == :terminated
-        bootstrap_options = bootstrap_options_for(action_handler, machine_spec, machine_options)
 
         action_handler.perform_action "Create #{machine_spec.name} with AMI #{bootstrap_options[:image_id]} in #{aws_config.region}" do
           Chef::Log.debug "Creating instance with bootstrap options #{bootstrap_options}"
 
-          instance = ec2.instances.create(bootstrap_options.to_hash)
+          actual_instance = ec2.instances.create(bootstrap_options.to_hash)
 
           # Make sure the instance is ready to be tagged
-          sleep 5 while instance.status == :pending
+          Retryable.retryable(:tries => 12, :sleep => 5, :on => [AWS::EC2::Errors::InvalidInstanceID::NotFound, TimeoutError]) do
+            raise TimeoutError unless actual_instance.status == :pending || actual_instance.status == :running
+          end
           # TODO add other tags identifying user / node url (same as fog)
-          instance.tags['Name'] = machine_spec.name
-          instance.source_dest_check = machine_options[:source_dest_check] if machine_options.has_key?(:source_dest_check)
+          actual_instance.tags['Name'] = machine_spec.name
+          actual_instance.source_dest_check = machine_options[:source_dest_check] if machine_options.has_key?(:source_dest_check)
           machine_spec.reference = {
               'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
               'allocated_at' => Time.now.utc.to_s,
               'host_node' => action_handler.host_node,
               'image_id' => bootstrap_options[:image_id],
-              'instance_id' => instance.id
+              'instance_id' => actual_instance.id
           }
           machine_spec.driver_url = driver_url
           machine_spec.reference['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
@@ -429,6 +459,9 @@ EOD
           end
         end
       end
+      # TODO because we don't want to add `provider_tags` as a base attribute,
+      # we have to update the tags here in driver.rb instead of the providers
+      converge_tags(actual_instance, machine_options[:aws_tags], action_handler)
     end
 
     def allocate_machines(action_handler, specs_and_options, parallelizer)
@@ -470,17 +503,15 @@ EOD
     end
 
     def destroy_machine(action_handler, machine_spec, machine_options)
-      instance = instance_for(machine_spec)
-      if instance && instance.exists?
-        # TODO do we need to wait_until(action_handler, machine_spec, instance) { instance.status != :shutting_down } ?
-        action_handler.perform_action "Terminate #{machine_spec.name} (#{machine_spec.reference['instance_id']}) in #{aws_config.region} ..." do
-          instance.terminate
-          machine_spec.reference = nil
+      d = self
+      Provisioning.inline_resource(action_handler) do
+        aws_instance machine_spec.name do
+          action :destroy
+          driver d
         end
-      else
-        Chef::Log.warn "Instance #{machine_spec.reference['instance_id']} doesn't exist for #{machine_spec.name}"
       end
 
+      # TODO move this into the aws_instance provider somehow
       strategy = convergence_strategy_for(machine_spec, machine_options)
       strategy.cleanup_convergence(action_handler, machine_spec)
     end
@@ -563,6 +594,7 @@ EOD
 
     def bootstrap_options_for(action_handler, machine_spec, machine_options)
       bootstrap_options = (machine_options[:bootstrap_options] || {}).to_h.dup
+      bootstrap_options[:instance_type] ||= default_instance_type
       image_id = bootstrap_options[:image_id] || machine_options[:image_id] || default_ami_for_region(aws_config.region)
       bootstrap_options[:image_id] = image_id
       if !bootstrap_options[:key_name]
@@ -650,23 +682,23 @@ EOD
 
       case region
         when 'ap-northeast-1'
-          'ami-c786dcc6'
+          'ami-6cbca76d'
         when 'ap-southeast-1'
-          'ami-eefca7bc'
+          'ami-04c6ec56'
         when 'ap-southeast-2'
-          'ami-996706a3'
+          'ami-c9eb9ff3'
         when 'eu-west-1'
-          'ami-4ab46b3d'
+          'ami-5f9e1028'
         when 'eu-central-1'
-          'ami-7c3c0a61'
+          'ami-56c2f14b'
         when 'sa-east-1'
-          'ami-6770d87a'
+          'ami-81f14e9c'
         when 'us-east-1'
-          'ami-d2ff23ba'
+          'ami-12793a7a'
         when 'us-west-1'
-          'ami-73717d36'
+          'ami-6ebca42b'
         when 'us-west-2'
-          'ami-f1ce8bc1'
+          'ami-b9471c89'
         else
           raise 'Unsupported region!'
       end
@@ -912,6 +944,7 @@ EOD
           if actual_instance.status == :terminated
             Chef::Log.warn "Machine #{machine_spec.name} (#{actual_instance.id}) is terminated.  Recreating ..."
           else
+            converge_tags(actual_instance, machine_options[:aws_tags], action_handler)
             yield machine_spec, actual_instance if block_given?
             next
           end
@@ -951,6 +984,7 @@ EOD
             machine_spec.driver_url = driver_url
             instance.tags['Name'] = machine_spec.name
             instance.source_dest_check = machine_options[:source_dest_check] if machine_options.has_key?(:source_dest_check)
+            converge_tags(instance, machine_options[:aws_tags], action_handler)
             machine_spec.reference['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
             %w(is_windows ssh_username sudo use_private_ip_for_ssh ssh_gateway).each do |key|
               machine_spec.reference[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
@@ -977,6 +1011,43 @@ EOD
       end.to_a
     end
 
+    # TODO This is currently duplicated from AWS Provider
+    # Set the tags on the aws object to desired_tags, while ignoring any `Name` tag
+    # If no tags need to be modified, will not perform a write call on AWS
+    def converge_tags(
+      aws_object,
+      desired_tags,
+      action_handler,
+      read_tags_block=lambda {|aws_object| aws_object.tags.to_h},
+      set_tags_block=lambda {|aws_object, desired_tags| aws_object.tags.set(desired_tags) },
+      delete_tags_block=lambda {|aws_object, tags_to_delete| aws_object.tags.delete(*tags_to_delete) }
+    )
+      # If aws_tags were not provided we exit
+      if desired_tags.nil?
+        Chef::Log.debug "aws_tags not provided, nothing to converge"
+        return
+      end
+      current_tags = read_tags_block.call(aws_object)
+      # AWS always returns tags as strings, and we don't want to overwrite a
+      # tag-as-string with the same tag-as-symbol
+      desired_tags = Hash[desired_tags.map {|k, v| [k.to_s, v.to_s] }]
+      tags_to_update = desired_tags.reject {|k,v| current_tags[k] == v}
+      tags_to_delete = current_tags.keys - desired_tags.keys
+      # We don't want to delete `Name`, just all other tags
+      tags_to_delete.delete('Name')
+
+      unless tags_to_update.empty?
+        action_handler.perform_action "applying tags #{tags_to_update}" do
+          set_tags_block.call(aws_object, tags_to_update)
+        end
+      end
+      unless tags_to_delete.empty?
+        action_handler.perform_action "deleting tags #{tags_to_delete.inspect}" do
+          delete_tags_block.call(aws_object, tags_to_delete)
+        end
+      end
+    end
+
     def get_listeners(listeners)
       case listeners
       when Hash
@@ -1035,7 +1106,7 @@ EOD
     end
 
     def default_instance_type
-      't1.micro'
+      't2.micro'
     end
 
     PORT_DEFAULTS = {

data/lib/chef/provisioning/aws_driver/exceptions.rb

@@ -0,0 +1,16 @@
+class Chef
+module Provisioning
+module AWSDriver
+module Exceptions
+
+  class MultipleSecurityGroupError < RuntimeError
+    def initialize(name, groups)
+      super "Found security groups with ids [#{groups.map {|sg| sg.id}}] that share name #{name}. " \
+        "Names are unique within VPCs - specify VPC to find by name."
+    end
+  end
+
+end
+end
+end
+end

data/lib/chef/provisioning/aws_driver/super_lwrp.rb

@@ -45,7 +45,7 @@ end
 
 module NoResourceCloning
   def prior_resource
-    if resource_class.kind_of?(Chef::Provisioning::AWSDriver::SuperLWRP)
+    if resource_class <= Chef::Provisioning::AWSDriver::SuperLWRP
       Chef::Log.debug "Canceling resource cloning for #{resource_class}"
       nil
     else

data/lib/chef/provisioning/aws_driver/version.rb

@@ -1,7 +1,7 @@
 class Chef
 module Provisioning
 module AWSDriver
-  VERSION = '1.1.1'
+  VERSION = '1.2.0'
 end
 end
 end

data/lib/chef/resource/aws_ebs_volume.rb

@@ -8,8 +8,8 @@ class Chef::Resource::AwsEbsVolume < Chef::Provisioning::AWSDriver::AWSResourceW
 
   attribute :machine,           kind_of: [ String, FalseClass, AwsInstance, AWS::EC2::Instance ]
 
-  attribute :availability_zone, kind_of: String
-  attribute :size,              kind_of: Integer
+  attribute :availability_zone, kind_of: String, default: 'a'
+  attribute :size,              kind_of: Integer, default: 8
   attribute :snapshot,          kind_of: String
 
   attribute :iops,              kind_of: Integer

data/lib/chef/resource/aws_eip_address.rb

@@ -6,6 +6,9 @@ class Chef::Resource::AwsEipAddress < Chef::Provisioning::AWSDriver::AWSResource
 
   attribute :name, kind_of: String, name_attribute: true
 
+  # guh - every other AWSResourceWithEntry accepts tags EXCEPT this one
+  undef_method(:aws_tags)
+
   # TODO network interface
   attribute :machine,          kind_of: [String, FalseClass]
   attribute :associate_to_vpc, kind_of: [TrueClass, FalseClass]

data/lib/chef/resource/aws_route_table.rb

@@ -34,6 +34,17 @@ class Chef::Resource::AwsRouteTable < Chef::Provisioning::AWSDriver::AWSResource
   #
   attribute :vpc,    kind_of: [ String, AwsVpc, AWS::EC2::VPC ], required: true
 
+  #
+  # Enable route propagation from one or more virtual private gateways
+  #
+  # The value should be an array of virtual private gateway ID:
+  # ```ruby
+  # virtual_private_gateways ['vgw-abcd1234', 'vgw-abcd5678']
+  # ```
+  #
+  attribute :virtual_private_gateways, kind_of: [ String, Array ],
+            coerce: proc { |v| [v].flatten }
+
   #
   # The routes for this route table.
   #
@@ -58,6 +69,26 @@ class Chef::Resource::AwsRouteTable < Chef::Provisioning::AWSDriver::AWSResource
   #
   attribute :routes, kind_of: Hash
 
+  #
+  # Regex to ignore one or more route targets.
+  #
+  # This is helpful when configuring HA NAT instances. If a NAT instance fails
+  # a auto-scaling group may launch a new NAT instance and update the route
+  # table accordingly. Chef provisioning should not attempt to change or remove
+  # this route.
+  #
+  # This attribute is specified as a regex since the full ID of the
+  # instance/network interface is not known ahead of time. In most cases the
+  # NAT instance route will point at a network interface attached to the NAT
+  # instance. The ID prefix for network interfaces is 'eni'. The following
+  # example shows how to ignore network interface routes.
+  #
+  # ```ruby
+  # ignore_route_targets ['^eni-']
+  # ```
+  attribute :ignore_route_targets, kind_of: [ String, Array ], default: [],
+            coerce: proc { |v| [v].flatten }
+
   attribute :route_table_id, kind_of: String, aws_id_attribute: true, lazy_default: proc {
     name =~ /^rtb-[a-f0-9]{8}$/ ? name : nil
   }