chef-provisioning-aws 1.4.1 → 1.5.0

data/lib/chef/provisioning/aws_driver/credentials.rb

@@ -87,6 +87,15 @@ module AWSDriver
       end
     end
 
+    def load_env_variables
+      if ENV["AWS_ACCESS_KEY_ID"] && ENV["AWS_SECRET_ACCESS_KEY"]
+        @credentials["default"] = {
+          aws_access_key_id: ENV["AWS_ACCESS_KEY_ID"],
+          aws_secret_access_key: ENV["AWS_SECRET_ACCESS_KEY"]
+        }
+      end
+    end
+
     def load_default
       config_file = ENV['AWS_CONFIG_FILE'] || File.expand_path('~/.aws/config')
       credentials_file = ENV['AWS_CREDENTIAL_FILE'] || File.expand_path('~/.aws/credentials')
@@ -97,6 +106,9 @@ module AWSDriver
           load_inis(config_file)
         end
       end
+      if @credentials.size == 0
+        load_env_variables
+      end
     end
 
     def self.method_missing(name, *args, &block)

data/lib/chef/provisioning/aws_driver/driver.rb

@@ -28,7 +28,13 @@ require 'base64'
 
 # loads the entire aws-sdk
 AWS.eager_autoload!
-AWS_V2_SERVICES = {"EC2" => "ec2", "S3" => "s3", "ElasticLoadBalancing" => "elb", "IAM" => "iam"}
+AWS_V2_SERVICES = {
+  "EC2" => "ec2",
+  "Route53" => "route53",
+  "S3" => "s3",
+  "ElasticLoadBalancing" => "elb",
+  "IAM" => "iam",
+}
 Aws.eager_autoload!(:services => AWS_V2_SERVICES.keys)
 
 # Need to load the resources after the SDK because `aws_sdk_types` can mess
@@ -53,6 +59,24 @@ class Resource
 end
 end
 
+require 'chef/provider/load_balancer'
+class Chef
+class Provider
+  class LoadBalancer
+    # We override this so we can specify a machine name as `i-123456`
+    # This is totally a hack until we move away from base resources
+    def get_machine_spec!(machine_name)
+      if machine_name =~ /^i-[a-f0-9]{8}$/
+        Struct.new(:name, :reference).new(machine_name, {'instance_id' => machine_name})
+      else
+        Chef::Log.debug "Getting machine spec for #{machine_name}"
+        Provisioning.chef_managed_entry_store(new_resource.chef_server).get!(:machine, machine_name)
+      end
+    end
+  end
+end
+end
+
 Chef::Provider::Machine.additional_machine_option_keys << :aws_tags
 Chef::Provider::MachineImage.additional_image_option_keys << :aws_tags
 Chef::Provider::LoadBalancer.additional_lb_option_keys << :aws_tags
@@ -66,7 +90,7 @@ module AWSDriver
     include Chef::Mixin::ShellOut
     include Chef::Mixin::DeepMerge
 
-    attr_reader :aws_config
+    attr_reader :aws_config, :aws_config_2
 
     # URL scheme:
     # aws:profilename:region
@@ -97,7 +121,7 @@ module AWSDriver
       # Right now we are supporting both V1 and V2, so we create 2 config sets
       credentials2 = Credentials2.new(:profile_name => profile_name)
       Chef::Config.chef_provisioning ||= {}
-      ::Aws.config.update(
+      @aws_config_2 = {
         credentials: credentials2.get_credentials,
         region: region || ENV["AWS_DEFAULT_REGION"] || credentials[:region],
         # TODO when we get rid of V1 replace the credentials class with something that knows how
@@ -105,7 +129,27 @@ module AWSDriver
         :http_proxy => credentials[:proxy_uri] || nil,
         logger: Chef::Log.logger,
         retry_limit: Chef::Config.chef_provisioning[:aws_retry_limit] || 5
-      )
+      }
+
+      driver = self
+      Chef::Resource::Machine.send(:define_method, :aws_object) do
+        resource = Chef::Resource::AwsInstance.new(name, nil)
+        resource.driver driver
+        resource.managed_entry_store Chef::Provisioning.chef_managed_entry_store
+        resource.aws_object
+      end
+      Chef::Resource::MachineImage.send(:define_method, :aws_object) do
+        resource = Chef::Resource::AwsImage.new(name, nil)
+        resource.driver driver
+        resource.managed_entry_store Chef::Provisioning.chef_managed_entry_store
+        resource.aws_object
+      end
+      Chef::Resource::LoadBalancer.send(:define_method, :aws_object) do
+        resource = Chef::Resource::AwsLoadBalancer.new(name, nil)
+        resource.driver driver
+        resource.managed_entry_store Chef::Provisioning.chef_managed_entry_store
+        resource.aws_object
+      end
     end
 
     def self.canonicalize_url(driver_url, config)
@@ -141,7 +185,10 @@ module AWSDriver
         updates << "  with tags #{lb_options[:aws_tags]}" if lb_options[:aws_tags]
 
         action_handler.perform_action updates do
-          actual_elb = elb.load_balancers.create(lb_spec.name, lb_options)
+          # IAM says the server certificate exists, but ELB throws this error
+          Chef::Provisioning::AWSDriver::AWSProvider.retry_with_backoff(AWS::ELB::Errors::CertificateNotFound) do
+            actual_elb = elb.load_balancers.create(lb_spec.name, lb_options)
+          end
 
           lb_spec.reference = {
             'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
@@ -157,22 +204,9 @@ module AWSDriver
         end
 
         # TODO: refactor this whole giant method into many smaller method calls
-        # TODO if we update scheme, we don't need to run any of the other updates.
-        # Also, if things aren't specified (such as machines / listeners), we
-        # need to grab them from the actual load balancer so we don't lose them.
-        # i.e. load_balancer 'blah' do
-        #   lb_options: { scheme: 'other_scheme' }
-        # end
-        # TODO we will leak the actual_elb if we fail to finish creating it
-        # Update scheme - scheme is immutable once set, so if it is changing we need to delete the old
-        # ELB and create a new one
         if lb_options[:scheme] && lb_options[:scheme].downcase != actual_elb.scheme
-          desc = ["  updating scheme to #{lb_options[:scheme]}"]
-          desc << "  WARN: scheme is immutable, so deleting and re-creating the ELB"
-          perform_action.call(desc) do
-            old_elb = actual_elb
-            actual_elb = elb.load_balancers.create(lb_spec.name, lb_options)
-          end
+          # TODO CloudFormation automatically recreates the load_balancer, we should too
+          raise "Scheme is immutable - you need to :destroy and :create the load_balancer to recreated it with the new scheme"
         end
 
         # Update security groups
@@ -248,12 +282,13 @@ module AWSDriver
                   load_balancer_name: actual_elb.name,
                   subnets: attach_subnets
                 )
-              rescue AWS::ELB::Errors::InvalidConfigurationRequest
-                raise "You cannot currently move from 1 subnet to another in the same availability zone. " +
+              rescue AWS::ELB::Errors::InvalidConfigurationRequest => e
+                Chef::Log.error "You cannot currently move from 1 subnet to another in the same availability zone. " +
                     "Amazon does not have an atomic operation which allows this.  You must create a new " +
                     "ELB with the correct subnets and move instances into it.  Tried to attach subets " +
                     "#{attach_subnets.join(', ')} (availability zones #{enable_zones.join(', ')}) to " +
                     "existing ELB named #{actual_elb.name}"
+                raise e
               end
             end
           end
@@ -591,11 +626,11 @@ EOD
       class_eval <<-META
 
       def #{short_name}_client
-        @#{short_name}_client ||= ::Aws::#{load_name}::Client.new
+        @#{short_name}_client ||= ::Aws::#{load_name}::Client.new(**aws_config_2)
       end
 
       def #{short_name}_resource
-        @#{short_name}_resource ||= ::Aws::#{load_name}::Resource.new(#{short_name}_client)
+        @#{short_name}_resource ||= ::Aws::#{load_name}::Resource.new(**(aws_config_2.merge({client: #{short_name}_client})))
       end
 
       META
@@ -621,10 +656,6 @@ EOD
       @s3 ||= AWS::S3.new(config: aws_config)
     end
 
-    def rds
-      @rds ||= AWS::RDS.new(config: aws_config)
-    end
-
     def sns
       @sns ||= AWS::SNS.new(config: aws_config)
     end
@@ -697,9 +728,6 @@ EOD
         Chef::Log.debug('No key specified, generating a default one...')
         bootstrap_options[:key_name] = default_aws_keypair(action_handler, machine_spec)
       end
-      if bootstrap_options[:iam_instance_profile] && bootstrap_options[:iam_instance_profile].is_a?(String)
-        bootstrap_options[:iam_instance_profile] = {name: bootstrap_options[:iam_instance_profile]}
-      end
       if bootstrap_options[:user_data]
         bootstrap_options[:user_data] = Base64.encode64(bootstrap_options[:user_data])
       end
@@ -727,6 +755,12 @@ EOD
 
       bootstrap_options = AWSResource.lookup_options(bootstrap_options, managed_entry_store: machine_spec.managed_entry_store, driver: self)
 
+      # We do this after the lookup_options because we need the aws_iam_instance_profile resource to
+      # only be passed a String during resource lookup, not `{name: ...}`
+      if bootstrap_options[:iam_instance_profile] && bootstrap_options[:iam_instance_profile].is_a?(String)
+        bootstrap_options[:iam_instance_profile] = {name: bootstrap_options[:iam_instance_profile]}
+      end
+
       # In the migration from V1 to V2 we still support associate_public_ip_address at the top level
       # we do this after the lookup because we have to copy any present subnets, etc. into the
       # network interfaces block
@@ -1007,7 +1041,7 @@ EOD
       image ||= image_for(image_spec)
       time_elapsed = 0
       sleep_time = 10
-      max_wait_time = 300
+      max_wait_time = Chef::Config.chef_provisioning[:image_max_wait_time] || 300
       if !yield(image)
         action_handler.report_progress "waiting for #{image_spec.name} (#{image.id} on #{driver_url}) to be ready ..."
         while time_elapsed < max_wait_time && !yield(image)
@@ -1030,8 +1064,7 @@ EOD
 
     def wait_until_machine(action_handler, machine_spec, output_msg, instance=nil, &block)
       instance ||= instance_for(machine_spec)
-      # TODO make these configurable from Chef::Config
-      max_attempts = 12
+      max_attempts = ((Chef::Config.chef_provisioning[:machine_max_wait_time] || 120) / 10).floor
       delay = 10
       log_progress = Proc.new do |attempts, response|
         action_handler.report_progress "been waiting #{delay*attempts}/#{delay*max_attempts} -- sleeping #{delay} seconds for #{machine_spec.name} (#{instance.id} on #{driver_url}) to #{output_msg} ..."
@@ -1054,7 +1087,7 @@ EOD
       instance = instance_for(machine_spec)
       time_elapsed = 0
       sleep_time = 10
-      max_wait_time = 120
+      max_wait_time = Chef::Config.chef_provisioning[:machine_max_wait_time] || 120
       transport = transport_for(machine_spec, machine_options, instance)
       unless transport.available?
         if action_handler.should_perform_actions
@@ -1178,12 +1211,24 @@ EOD
     end
 
     def create_instance_and_reference(bootstrap_options, action_handler, machine_spec, machine_options)
-      instance = ec2_resource.create_instances(bootstrap_options.to_hash)[0]
+      instance = nil
+      # IAM says the instance profile is ready, but EC2 doesn't think it is
+      # Not using retry_with_backoff here because we need to match on a string
+      Retryable.retryable(
+        :tries => 10,
+        :sleep => lambda { |n| [2**n, 16].min },
+        :on => ::Aws::EC2::Errors::InvalidParameterValue,
+        :matching => /Invalid IAM Instance Profile name/
+      ) do |retries, exception|
+        Chef::Log.debug("Instance creation InvalidParameterValue exception is #{exception.inspect}")
+        instance = ec2_resource.create_instances(bootstrap_options.to_hash)[0]
+      end
+
       # Make sure the instance is ready to be tagged
       instance.wait_until_exists
 
       # Sometimes tagging fails even though the instance 'exists'
-      Retryable.retryable(:tries => 12, :sleep => 5, :on => [::Aws::EC2::Errors::InvalidInstanceIDNotFound]) do
+      Chef::Provisioning::AWSDriver::AWSProvider.retry_with_backoff(::Aws::EC2::Errors::InvalidInstanceIDNotFound) do
         instance.create_tags({tags: [{key: "Name", value: machine_spec.name}]})
       end
       if machine_options.has_key?(:source_dest_check)

data/lib/chef/provisioning/aws_driver/super_lwrp.rb

@@ -5,60 +5,73 @@ module Provisioning
 module AWSDriver
   class SuperLWRP < Chef::Resource::LWRPBase
     #
-    # Add the :lazy_default and :coerce validation_opts to `attribute`
+    # Add the :default lazy { ... } and :coerce validation_opts to `attribute`
     #
-    def self.attribute(attr_name, validation_opts={})
-      lazy_default = validation_opts.delete(:lazy_default)
-      coerce = validation_opts.delete(:coerce)
-      if lazy_default || coerce
-        define_method(attr_name) do |arg=nil|
-          arg = instance_exec(arg, &coerce) if coerce && !arg.nil?
+    if self.respond_to?(:properties)
+      # in Chef 12.5+, properties replace attributes and these respond to
+      # coerce and default with a lazy block - no need for overwriting!
+    else
+      def self.attribute(attr_name, validation_opts={})
+        if validation_opts[:default].is_a?(Chef::DelayedEvaluator)
+          lazy_default = validation_opts.delete(:default)
+        end
+        coerce = validation_opts.delete(:coerce)
+        if lazy_default || coerce
+          define_method(attr_name) do |arg=nil|
+            arg = instance_exec(arg, &coerce) if coerce && !arg.nil?
 
-          result = set_or_return(attr_name.to_sym, arg, validation_opts)
+            result = set_or_return(attr_name.to_sym, arg, validation_opts)
 
-          if result.nil? && arg.nil?
-            result = instance_eval(&lazy_default) if lazy_default
-          end
+            if result.nil? && arg.nil?
+              result = instance_eval(&lazy_default) if lazy_default
+            end
 
-          result
-        end
-        define_method(:"#{attr_name}=") do |arg|
-          if arg.nil?
-            remove_instance_variable(:"@#{arg}")
-          else
-            set_or_return(attr_name.to_sym, arg, validation_opts)
+            result
+          end
+          define_method(:"#{attr_name}=") do |arg|
+            if arg.nil?
+              remove_instance_variable(:"@#{arg}")
+            else
+              set_or_return(attr_name.to_sym, arg, validation_opts)
+            end
           end
+        else
+          super
         end
-      else
-        super
       end
-    end
 
-    # FUUUUUU cloning - this works for Chef 11 or 12.1
-    def load_prior_resource(*args)
-      Chef::Log.debug "Overloading #{self.resource_name} load_prior_resource with NOOP"
+      # Below chef 12.5 you cannot do `default lazy: { ... }` - this adds that
+      def self.lazy(&block)
+        Chef::DelayedEvaluator.new(&block)
+      end
     end
-  end
-end
-end
-end
 
-module NoResourceCloning
-  def prior_resource
-    if resource_class <= Chef::Provisioning::AWSDriver::SuperLWRP
-      Chef::Log.debug "Canceling resource cloning for #{resource_class}"
-      nil
-    else
-      super
+    # copy from Chef 12.5 params_validate.rb at http://redirx.me/?t35q.
+    if !method_defined?(:_pv_is)
+      def _pv_is(opts, key, to_be, raise_error: true)
+        return true if !opts.has_key?(key.to_s) && !opts.has_key?(key.to_sym)
+        value = _pv_opts_lookup(opts, key)
+        to_be = [ to_be ].flatten(1)
+        to_be.each do |tb|
+          case tb
+          when Proc
+            return true if instance_exec(value, &tb)
+          when Property
+            validate(opts, { key => tb.validation_options })
+            return true
+          else
+            return true if tb === value
+          end
+        end
+
+        if raise_error
+          raise ::Chef::Exceptions::ValidationFailed, "Option #{key} must be one of: #{to_be.join(", ")}!  You passed #{value.inspect}."
+        else
+          false
+        end
+      end
     end
   end
-  def emit_cloned_resource_warning; end
-  def emit_harmless_cloning_debug; end
 end
-
-# Chef 12.2 changed `load_prior_resource` logic to be in the Chef::ResourceBuilder class
-# but that class only exists in 12.2 and up
-if defined? Chef::ResourceBuilder
-  # Ruby 2.0.0 has prepend as a protected method
-  Chef::ResourceBuilder.send(:prepend, NoResourceCloning)
+end
 end

data/lib/chef/provisioning/aws_driver/version.rb

@@ -1,7 +1,7 @@
 class Chef
 module Provisioning
 module AWSDriver
-  VERSION = '1.4.1'
+  VERSION = '1.5.0'
 end
 end
 end

data/lib/chef/resource/aws_dhcp_options.rb

@@ -47,7 +47,7 @@ class Chef::Resource::AwsDhcpOptions < Chef::Provisioning::AWSDriver::AWSResourc
   #
   attribute :netbios_node_type, kind_of: Integer
 
-  attribute :dhcp_options_id, kind_of: String, aws_id_attribute: true, lazy_default: proc {
+  attribute :dhcp_options_id, kind_of: String, aws_id_attribute: true, default: lazy {
     name =~ /^dopt-[a-f0-9]{8}$/ ? name : nil
   }
 

data/lib/chef/resource/aws_ebs_volume.rb

@@ -19,7 +19,7 @@ class Chef::Resource::AwsEbsVolume < Chef::Provisioning::AWSDriver::AWSResourceW
   attribute :encrypted,         kind_of: [ TrueClass, FalseClass ]
   attribute :device,            kind_of: String
 
-  attribute :volume_id,         kind_of: String, aws_id_attribute: true, lazy_default: proc {
+  attribute :volume_id,         kind_of: String, aws_id_attribute: true, default: lazy {
     name =~ /^vol-[a-f0-9]{8}$/ ? name : nil
   }
 

data/lib/chef/resource/aws_eip_address.rb

@@ -26,7 +26,7 @@ class Chef::Resource::AwsEipAddress < Chef::Provisioning::AWSDriver::AWSResource
   # ```
   #
   attribute :public_ip, kind_of: String, aws_id_attribute: true, coerce: proc { |v| IPAddr.new(v); v },
-    lazy_default: proc {
+    default: lazy {
       begin
         IPAddr.new(name)
         name

data/lib/chef/resource/aws_iam_instance_profile.rb

@@ -0,0 +1,33 @@
+require 'chef/provisioning/aws_driver/aws_resource'
+
+#
+# An AWS IAM instance profile, a container for an IAM role that you can use to
+# pass role information to an EC2 instance when the instance starts..
+#
+# `name` is unique for an AWS account.
+#
+# API documentation for the AWS Ruby SDK for IAM instance profiles (and the object returned from `aws_object`) can be found here:
+#
+# - http://docs.aws.amazon.com/sdkforruby/api/Aws/IAM/InstanceProfile.html
+#
+class Chef::Resource::AwsIamInstanceProfile < Chef::Provisioning::AWSDriver::AWSResource
+  aws_sdk_type ::Aws::IAM::InstanceProfile
+
+  #
+  # The name of the instance profile to create.
+  #
+  attribute :name,   kind_of: String, name_attribute: true
+
+  #
+  # The path to the instance profile. For more information about paths, see http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html
+  #
+  attribute :path,    kind_of: String
+
+  attribute :role,    kind_of: [ String, AwsIamRole, ::Aws::IAM::Role]
+
+  def aws_object
+    result = driver.iam_resource.instance_profile(name)
+    result && result.exists? ? result : nil
+  end
+
+end