chef-provisioning-aws 1.3.1 → 1.4.0

data/lib/chef/provisioning/aws_driver/credentials2.rb

@@ -0,0 +1,51 @@
+require "aws-sdk"
+require "aws-sdk-core/credentials"
+require "aws-sdk-core/shared_credentials"
+require "aws-sdk-core/instance_profile_credentials"
+require "aws-sdk-core/assume_role_credentials"
+
+class Chef
+module Provisioning
+module AWSDriver
+
+  class LoadCredentialsError < RuntimeError; end
+
+  # Loads the credentials for the AWS SDK V2
+  # Attempts to load credentials in the order specified at http://docs.aws.amazon.com/sdkforruby/api/index.html#Configuration
+  class Credentials2
+
+    attr_reader :profile_name
+
+    # @param [Hash] options
+    # @option options [String] :profile_name (ENV["AWS_DEFAULT_PROFILE"]) The profile name to use
+    #    when loading the config from '~/.aws/credentials'.  This can be nil.
+    def initialize(options = {})
+      @profile_name = options[:profile_name] || ENV["AWS_DEFAULT_PROFILE"]
+    end
+
+    # Try to load the credentials from an ordered list of sources and return the first one that
+    # can be loaded successfully.
+    def get_credentials
+      shared_creds = ::Aws::SharedCredentials.new(:profile_name => profile_name)
+      instance_profile_creds = ::Aws::InstanceProfileCredentials.new(:retries => 1)
+
+      if ENV["AWS_ACCESS_KEY_ID"] && ENV["AWS_SECRET_ACCESS_KEY"]
+        creds = ::Aws::Credentials.new(
+          ENV["AWS_ACCESS_KEY_ID"],
+          ENV["AWS_SECRET_ACCESS_KEY"],
+          ENV["AWS_SESSION_TOKEN"]
+        )
+      elsif shared_creds.set?
+        creds = shared_creds
+      elsif instance_profile_creds.set?
+        creds = instance_profile_creds
+      else
+        raise LoadCredentialsError.new("Could not load credentials from the environment variables, the .aws/credentials file or the metadata service")
+      end
+      creds
+    end
+  end
+
+end
+end
+end

data/lib/chef/provisioning/aws_driver/driver.rb

@@ -11,21 +11,50 @@ require 'chef/provisioning/machine/windows_machine'
 require 'chef/provisioning/machine/unix_machine'
 require 'chef/provisioning/machine_spec'
 
-require 'chef/resource/aws_key_pair'
-require 'chef/resource/aws_instance'
-require 'chef/resource/aws_image'
-require 'chef/resource/aws_load_balancer'
 require 'chef/provisioning/aws_driver/aws_resource'
+require 'chef/provisioning/aws_driver/tagging_strategy/ec2'
+require 'chef/provisioning/aws_driver/tagging_strategy/elb'
 require 'chef/provisioning/aws_driver/version'
 require 'chef/provisioning/aws_driver/credentials'
+require 'chef/provisioning/aws_driver/credentials2'
+require 'chef/provisioning/aws_driver/aws_tagger'
 
 require 'yaml'
 require 'aws-sdk-v1'
+require 'aws-sdk'
 require 'retryable'
 require 'ubuntu_ami'
 
 # loads the entire aws-sdk
 AWS.eager_autoload!
+AWS_V2_SERVICES = {"EC2" => "ec2", "S3" => "s3", "ElasticLoadBalancing" => "elb"}
+Aws.eager_autoload!(:services => AWS_V2_SERVICES.keys)
+
+# Need to load the resources after the SDK because `aws_sdk_types` can mess
+# up AWS loading if they are loaded too early
+require 'chef/resource/aws_key_pair'
+require 'chef/resource/aws_instance'
+require 'chef/resource/aws_image'
+require 'chef/resource/aws_load_balancer'
+
+# We add the appropriate attributes to the base resources for tagging support
+class Chef
+class Resource
+  class Machine
+    include Chef::Provisioning::AWSDriver::AWSTaggable
+  end
+  class MachineImage
+    include Chef::Provisioning::AWSDriver::AWSTaggable
+  end
+  class LoadBalancer
+    include Chef::Provisioning::AWSDriver::AWSTaggable
+  end
+end
+end
+
+Chef::Provider::Machine.additional_machine_option_keys << :aws_tags
+Chef::Provider::MachineImage.additional_image_option_keys << :aws_tags
+Chef::Provider::LoadBalancer.additional_lb_option_keys << :aws_tags
 
 class Chef
 module Provisioning
@@ -62,6 +91,20 @@ module AWSDriver
         session_token: credentials[:aws_session_token] || nil,
         logger: Chef::Log.logger
       )
+
+      # TODO document how users could add something to the Aws.config themselves if they want to
+      # Right now we are supporting both V1 and V2, so we create 2 config sets
+      credentials2 = Credentials2.new(:profile_name => profile_name)
+      Chef::Config.chef_provisioning ||= {}
+      ::Aws.config.update(
+        credentials: credentials2.get_credentials,
+        region: region || ENV["AWS_DEFAULT_REGION"] || credentials[:region],
+        # TODO when we get rid of V1 replace the credentials class with something that knows how
+        # to read ~/.aws/config
+        :http_proxy => credentials[:proxy_uri] || nil,
+        logger: Chef::Log.logger,
+        retry_limit: Chef::Config.chef_provisioning[:aws_retry_limit] || 5
+      )
     end
 
     def self.canonicalize_url(driver_url, config)
@@ -70,10 +113,13 @@ module AWSDriver
 
     # Load balancer methods
     def allocate_load_balancer(action_handler, lb_spec, lb_options, machine_specs)
-      lb_options = AWSResource.lookup_options(lb_options || {}, managed_entry_store: lb_spec.managed_entry_store, driver: self)
-      # We delete the attributes here because they are not valid in the create call
+      lb_options = (lb_options || {}).to_h
+      lb_options = AWSResource.lookup_options(lb_options, managed_entry_store: lb_spec.managed_entry_store, driver: self)
+      # We delete the attributes and a health check here because they are not valid in the create call
       # and must be applied afterward
       lb_attributes = lb_options.delete(:attributes)
+      lb_aws_tags = lb_options.delete(:aws_tags)
+      health_check  = lb_options.delete(:health_check)
 
       old_elb = nil
       actual_elb = load_balancer_for(lb_spec)
@@ -93,12 +139,8 @@ module AWSDriver
         updates << "  with security groups #{lb_options[:security_groups]}" if lb_options[:security_groups]
         updates << "  with tags #{lb_options[:aws_tags]}" if lb_options[:aws_tags]
 
-
-        lb_aws_tags = lb_options[:aws_tags]
-        lb_options.delete(:aws_tags)
         action_handler.perform_action updates do
           actual_elb = elb.load_balancers.create(lb_spec.name, lb_options)
-          lb_options[:aws_tags] = lb_aws_tags
 
           lb_spec.reference = {
             'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
@@ -281,34 +323,7 @@ module AWSDriver
         end
       end
 
-      # GRRRR curse you AWS and your crappy tagging support for ELBs
-      read_tags_block = lambda {|aws_object|
-        resp = elb.client.describe_tags load_balancer_names: [aws_object.name]
-        tags = {}
-        resp.data[:tag_descriptions] && resp.data[:tag_descriptions].each do |td|
-          td[:tags].each do |t|
-            tags[t[:key]] = t[:value]
-          end
-        end
-        tags
-      }
-
-      set_tags_block = lambda {|aws_object, desired_tags|
-        aws_form_tags = []
-        desired_tags.each do |k, v|
-          aws_form_tags << {key: k, value: v}
-        end
-        elb.client.add_tags load_balancer_names: [aws_object.name], tags: aws_form_tags
-      }
-
-      delete_tags_block=lambda {|aws_object, tags_to_delete|
-        aws_form_tags = []
-        tags_to_delete.each do |k, v|
-          aws_form_tags << {key: k}
-        end
-        elb.client.remove_tags load_balancer_names: [aws_object.name], tags: aws_form_tags
-      }
-      converge_tags(actual_elb, lb_options[:aws_tags], action_handler, read_tags_block, set_tags_block, delete_tags_block)
+      converge_elb_tags(actual_elb, lb_aws_tags, action_handler)
 
       # Update load balancer attributes
       if lb_attributes
@@ -325,12 +340,26 @@ module AWSDriver
         end
       end
 
+      # Update the load balancer health check, as above
+      if health_check
+        current = elb.client.describe_load_balancers(load_balancer_names: [actual_elb.name])[:load_balancer_descriptions][0][:health_check]
+        desired = deep_merge!(health_check, Marshal.load(Marshal.dump(current)))
+        if current != desired
+          perform_action.call("  updating health check to #{desired.inspect}") do
+            elb.client.configure_health_check(
+              load_balancer_name: actual_elb.name,
+              health_check: desired.to_hash
+            )
+          end
+        end
+      end
+
       # Update instance list, but only if there are machines specified
       if machine_specs
-        actual_instance_ids = actual_elb.instances.map { |i| i.instance_id }
+        assigned_instance_ids = actual_elb.instances.map { |i| i.instance_id }
 
-        instances_to_add = machine_specs.select { |s| !actual_instance_ids.include?(s.reference['instance_id']) }
-        instance_ids_to_remove = actual_instance_ids - machine_specs.map { |s| s.reference['instance_id'] }
+        instances_to_add = machine_specs.select { |s| !assigned_instance_ids.include?(s.reference['instance_id']) }
+        instance_ids_to_remove = assigned_instance_ids - machine_specs.map { |s| s.reference['instance_id'] }
 
         if instances_to_add.size > 0
           perform_action.call("  add machines #{instances_to_add.map { |s| s.name }.join(', ')}") do
@@ -410,6 +439,8 @@ module AWSDriver
     # Image methods
     def allocate_image(action_handler, image_spec, image_options, machine_spec, machine_options)
       actual_image = image_for(image_spec)
+      image_options = (image_options || {}).to_h.dup
+      machine_options = (machine_options || {}).to_h.dup
       aws_tags = image_options.delete(:aws_tags) || {}
       if actual_image.nil? || !actual_image.exists? || actual_image.state == :failed
         action_handler.perform_action "Create image #{image_spec.name} from machine #{machine_spec.name} with options #{image_options.inspect}" do
@@ -421,13 +452,14 @@ module AWSDriver
           image_spec.reference = {
             'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
             'image_id' => actual_image.id,
-            'allocated_at' => Time.now.to_i
+            'allocated_at' => Time.now.to_i,
+            'from-instance' => image_options[:instance_id]
           }
           image_spec.driver_url = driver_url
         end
       end
-      aws_tags['From-Instance'] = image_options[:instance_id] if image_options[:instance_id]
-      converge_tags(actual_image, aws_tags, action_handler)
+      aws_tags['from-instance'] = image_options[:instance_id] if image_options[:instance_id]
+      converge_ec2_tags(actual_image, aws_tags, action_handler)
     end
 
     def ready_image(action_handler, image_spec, image_options)
@@ -435,11 +467,13 @@ module AWSDriver
       if actual_image.nil? || !actual_image.exists?
         raise 'Cannot ready an image that does not exist'
       else
+        image_options = (image_options || {}).to_h.dup
+        aws_tags = image_options.delete(:aws_tags) || {}
+        aws_tags['from-instance'] = image_spec.reference['from-instance'] if image_spec.reference['from-instance']
+        converge_ec2_tags(actual_image, aws_tags, action_handler)
         if actual_image.state != :available
           action_handler.report_progress 'Waiting for image to be ready ...'
           wait_until_ready_image(action_handler, image_spec, actual_image)
-        else
-          action_handler.report_progress "Image #{image_spec.name} is ready!"
         end
       end
     end
@@ -477,38 +511,16 @@ EOD
 
     # Machine methods
     def allocate_machine(action_handler, machine_spec, machine_options)
-      actual_instance = instance_for(machine_spec)
+      instance = instance_for(machine_spec)
       bootstrap_options = bootstrap_options_for(action_handler, machine_spec, machine_options)
 
-      if actual_instance == nil || !actual_instance.exists? || actual_instance.status == :terminated
-
+      if instance == nil || !instance.exists? || instance.state.name == "terminated"
         action_handler.perform_action "Create #{machine_spec.name} with AMI #{bootstrap_options[:image_id]} in #{aws_config.region}" do
           Chef::Log.debug "Creating instance with bootstrap options #{bootstrap_options}"
-
-          actual_instance = ec2.instances.create(bootstrap_options.to_hash)
-          # Make sure the instance is ready to be tagged
-          wait_until_taggable(actual_instance)
-
-          # TODO add other tags identifying user / node url (same as fog)
-          actual_instance.tags['Name'] = machine_spec.name
-          actual_instance.source_dest_check = machine_options[:source_dest_check] if machine_options.has_key?(:source_dest_check)
-          machine_spec.reference = {
-              'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
-              'allocated_at' => Time.now.utc.to_s,
-              'host_node' => action_handler.host_node,
-              'image_id' => bootstrap_options[:image_id],
-              'instance_id' => actual_instance.id
-          }
-          machine_spec.driver_url = driver_url
-          machine_spec.reference['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
-          %w(is_windows ssh_username sudo use_private_ip_for_ssh ssh_gateway).each do |key|
-            machine_spec.reference[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
-          end
+          instance = create_instance_and_reference(bootstrap_options, action_handler, machine_spec, machine_options)
         end
       end
-      # TODO because we don't want to add `provider_tags` as a base attribute,
-      # we have to update the tags here in driver.rb instead of the providers
-      converge_tags(actual_instance, machine_options[:aws_tags], action_handler)
+      converge_ec2_tags(instance, machine_options[:aws_tags], action_handler)
     end
 
     def allocate_machines(action_handler, specs_and_options, parallelizer)
@@ -520,14 +532,15 @@ EOD
 
     def ready_machine(action_handler, machine_spec, machine_options)
       instance = instance_for(machine_spec)
+      converge_ec2_tags(instance, machine_options[:aws_tags], action_handler)
 
       if instance.nil?
         raise "Machine #{machine_spec.name} does not have an instance associated with it, or instance does not exist."
       end
 
-      if instance.status != :running
-        wait_until_machine(action_handler, machine_spec, instance) { instance.status != :stopping }
-        if instance.status == :stopped
+      if instance.state.name != "running"
+        wait_until_machine(action_handler, machine_spec, "finish stopping", instance) { instance.state.name != "stopping" }
+        if instance.state.name == "stopped"
           action_handler.perform_action "Start #{machine_spec.name} (#{machine_spec.reference['instance_id']}) in #{aws_config.region} ..." do
             instance.start
           end
@@ -563,10 +576,30 @@ EOD
       strategy.cleanup_convergence(action_handler, machine_spec)
     end
 
+    def cloudsearch(api_version="20130101")
+      @cloudsearch ||= {}
+      @cloudsearch[api_version] ||= AWS::CloudSearch::Client.const_get("V#{api_version}").new
+      @cloudsearch[api_version]
+    end
+
     def ec2
       @ec2 ||= AWS::EC2.new(config: aws_config)
     end
 
+    AWS_V2_SERVICES.each do |load_name, short_name|
+      class_eval <<-META
+
+      def #{short_name}_client
+        @#{short_name}_client ||= ::Aws::#{load_name}::Client.new
+      end
+
+      def #{short_name}_resource
+        @#{short_name}_resource ||= ::Aws::#{load_name}::Resource.new(#{short_name}_client)
+      end
+
+      META
+    end
+
     def elb
       @elb ||= AWS::ELB.new(config: aws_config)
     end
@@ -579,10 +612,18 @@ EOD
       @iam ||= AWS::IAM.new(config: aws_config)
     end
 
+    def rds
+      @rds ||= AWS::RDS.new(config: aws_config)
+    end
+
     def s3
       @s3 ||= AWS::S3.new(config: aws_config)
     end
 
+    def rds
+      @rds ||= AWS::RDS.new(config: aws_config)
+    end
+
     def sns
       @sns ||= AWS::SNS.new(config: aws_config)
     end
@@ -645,8 +686,10 @@ EOD
 
     def bootstrap_options_for(action_handler, machine_spec, machine_options)
       bootstrap_options = (machine_options[:bootstrap_options] || {}).to_h.dup
+      # These are hardcoded for now - only 1 machine at a time
+      bootstrap_options[:min_count] = bootstrap_options[:max_count] = 1
       bootstrap_options[:instance_type] ||= default_instance_type
-      image_id = bootstrap_options[:image_id] || machine_options[:image_id] || default_ami_for_region(aws_config.region)
+      image_id = machine_options[:from_image] || bootstrap_options[:image_id] || machine_options[:image_id] || default_ami_for_region(aws_config.region)
       bootstrap_options[:image_id] = image_id
       if !bootstrap_options[:key_name]
         Chef::Log.debug('No key specified, generating a default one...')
@@ -834,10 +877,19 @@ EOD
     end
 
     def determine_remote_host(machine_spec, instance)
+      transport_address_location = (machine_spec.reference['transport_address_location'] || :none).to_sym
       if machine_spec.reference['use_private_ip_for_ssh']
+        # The machine_spec has the old config key, lets update it - a successful chef converge will save the machine_spec
+        # TODO in 2.0 get rid of this update
+        machine_spec.reference.delete('use_private_ip_for_ssh')
+        machine_spec.reference['transport_address_location'] = :private_ip
         instance.private_ip_address
-      elsif !instance.public_ip_address
-        Chef::Log.warn("Server #{machine_spec.name} has no public ip address.  Using private ip '#{instance.private_ip_address}'.  Set driver option 'use_private_ip_for_ssh' => true if this will always be the case ...")
+      elsif transport_address_location == :private_ip
+        instance.private_ip_address
+      elsif transport_address_location == :dns
+        instance.dns_name
+      elsif !instance.public_ip_address && instance.private_ip_address
+        Chef::Log.warn("Server #{machine_spec.name} has no public ip address.  Using private ip '#{instance.private_ip_address}'.  Set machine_options ':transport_address_location => :private_ip' if this will always be the case ...")
         instance.private_ip_address
       elsif instance.public_ip_address
         instance.public_ip_address
@@ -924,28 +976,31 @@ EOD
     end
 
     def wait_until_ready_machine(action_handler, machine_spec, instance=nil)
-      wait_until_machine(action_handler, machine_spec, instance) { instance.status == :running }
+      wait_until_machine(action_handler, machine_spec, "be ready", instance) { |instance|
+        instance.state.name == "running"
+      }
     end
 
-    def wait_until_machine(action_handler, machine_spec, instance=nil, &block)
+    def wait_until_machine(action_handler, machine_spec, output_msg, instance=nil, &block)
       instance ||= instance_for(machine_spec)
-      time_elapsed = 0
-      sleep_time = 10
-      max_wait_time = 120
-      if !yield(instance)
-        if action_handler.should_perform_actions
-          action_handler.report_progress "waiting for #{machine_spec.name} (#{instance.id} on #{driver_url}) to be ready ..."
-          while time_elapsed < max_wait_time && !yield(instance)
-            action_handler.report_progress "been waiting #{time_elapsed}/#{max_wait_time} -- sleeping #{sleep_time} seconds for #{machine_spec.name} (#{instance.id} on #{driver_url}) to be ready ..."
-            sleep(sleep_time)
-            time_elapsed += sleep_time
-          end
-          unless yield(instance)
-            raise "Image #{instance.id} did not become ready within #{max_wait_time} seconds"
+      # TODO make these configurable from Chef::Config
+      max_attempts = 12
+      delay = 10
+      log_progress = Proc.new do |attempts, response|
+        action_handler.report_progress "been waiting #{delay*attempts}/#{delay*max_attempts} -- sleeping #{delay} seconds for #{machine_spec.name} (#{instance.id} on #{driver_url}) to #{output_msg} ..."
+      end
+      if action_handler.should_perform_actions
+        no_wait = yield(instance)
+        unless no_wait
+          action_handler.report_progress "waiting for #{machine_spec.name} (#{instance.id} on #{driver_url}) to #{output_msg} ..."
+          instance.wait_until(:max_attempts => max_attempts, :delay => delay, before_wait: log_progress) do |instance|
+            yield(instance)
           end
-          action_handler.report_progress "#{machine_spec.name} is now ready"
         end
       end
+      # We need an instance.reload here because the `wait_until` does not reload the instance it is called on,
+      # only the instance that is passed to the block
+      instance.reload
     end
 
     def wait_for_transport(action_handler, machine_spec, machine_options)
@@ -996,18 +1051,20 @@ EOD
       default_key_name
     end
 
-    def create_servers(action_handler, specs_and_options, parallelizer, &block)
+    def create_servers(action_handler, specs_and_options, parallelizer)
       specs_and_servers = instances_for(specs_and_options.keys)
 
       by_bootstrap_options = {}
       specs_and_options.each do |machine_spec, machine_options|
-        actual_instance = specs_and_servers[machine_spec]
-        if actual_instance
-          if actual_instance.status == :terminated
-            Chef::Log.warn "Machine #{machine_spec.name} (#{actual_instance.id}) is terminated.  Recreating ..."
+        instance = specs_and_servers[machine_spec]
+        if instance
+          if instance.state.name == "terminated"
+            Chef::Log.warn "Machine #{machine_spec.name} (#{instance.id}) is terminated.  Recreating ..."
           else
-            converge_tags(actual_instance, machine_options[:aws_tags], action_handler)
-            yield machine_spec, actual_instance if block_given?
+            # Even though the instance has been created the tags could be incorrect if it
+            # was created before tags were introduced
+            converge_ec2_tags(instance, machine_options[:aws_tags], action_handler)
+            yield machine_spec, instance if block_given?
             next
           end
         elsif machine_spec.reference
@@ -1031,30 +1088,20 @@ EOD
         action_handler.report_progress description
         if action_handler.should_perform_actions
           # Actually create the servers
-          create_many_instances(machine_specs.size, bootstrap_options, parallelizer) do |instance|
+          parallelizer.parallelize(1.upto(machine_specs.size)) do |i|
 
             # Assign each one to a machine spec
             machine_spec = machine_specs.pop
             machine_options = specs_and_options[machine_spec]
-            machine_spec.reference = {
-              'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
-              'allocated_at' => Time.now.utc.to_s,
-              'host_node' => action_handler.host_node,
-              'image_id' => bootstrap_options[:image_id],
-              'instance_id' => instance.id
-            }
-            machine_spec.driver_url = driver_url
-            instance.tags['Name'] = machine_spec.name
-            instance.source_dest_check = machine_options[:source_dest_check] if machine_options.has_key?(:source_dest_check)
-            converge_tags(instance, machine_options[:aws_tags], action_handler)
-            machine_spec.reference['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
-            %w(is_windows ssh_username sudo use_private_ip_for_ssh ssh_gateway).each do |key|
-              machine_spec.reference[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
-            end
+
+            clean_bootstrap_options = Marshal.load(Marshal.dump(bootstrap_options))
+            instance = create_instance_and_reference(clean_bootstrap_options, action_handler, machine_spec, machine_options)
+            converge_ec2_tags(instance, machine_options[:aws_tags], action_handler)
+
             action_handler.performed_action "machine #{machine_spec.name} created as #{instance.id} on #{driver_url}"
 
             yield machine_spec, instance if block_given?
-          end
+          end.to_a
 
           if machine_specs.size > 0
             raise "Not all machines were created by create_servers"
@@ -1063,58 +1110,63 @@ EOD
       end.to_a
     end
 
-    def create_many_instances(num_servers, bootstrap_options, parallelizer)
-      parallelizer.parallelize(1.upto(num_servers)) do |i|
-        clean_bootstrap_options = Marshal.load(Marshal.dump(bootstrap_options))
-        instance = ec2.instances.create(clean_bootstrap_options.to_hash)
-        wait_until_taggable(instance)
+    def converge_ec2_tags(aws_object, tags, action_handler)
+      ec2_strategy = Chef::Provisioning::AWSDriver::TaggingStrategy::EC2.new(
+        ec2_client,
+        aws_object.id,
+        tags
+      )
+      aws_tagger = Chef::Provisioning::AWSDriver::AWSTagger.new(ec2_strategy, action_handler)
+      aws_tagger.converge_tags
+    end
 
-        yield instance if block_given?
-        instance
-      end.to_a
+    def converge_elb_tags(aws_object, tags, action_handler)
+      elb_strategy = Chef::Provisioning::AWSDriver::TaggingStrategy::ELB.new(
+        elb_client,
+        aws_object.name,
+        tags
+      )
+      aws_tagger = Chef::Provisioning::AWSDriver::AWSTagger.new(elb_strategy, action_handler)
+      aws_tagger.converge_tags
     end
 
-    # TODO This is currently duplicated from AWS Provider
-    # Set the tags on the aws object to desired_tags, while ignoring any `Name` tag
-    # If no tags need to be modified, will not perform a write call on AWS
-    def converge_tags(
-      aws_object,
-      desired_tags,
-      action_handler,
-      read_tags_block=lambda {|aws_object| aws_object.tags.to_h},
-      set_tags_block=lambda {|aws_object, desired_tags| aws_object.tags.set(desired_tags) },
-      delete_tags_block=lambda {|aws_object, tags_to_delete| aws_object.tags.delete(*tags_to_delete) }
-    )
-      # If aws_tags were not provided we exit
-      if desired_tags.nil?
-        Chef::Log.debug "aws_tags not provided, nothing to converge"
-        return
+    def create_instance_and_reference(bootstrap_options, action_handler, machine_spec, machine_options)
+      instance = ec2_resource.create_instances(bootstrap_options.to_hash)[0]
+      # Make sure the instance is ready to be tagged
+      instance.wait_until_exists
+
+      # Sometimes tagging fails even though the instance 'exists'
+      Retryable.retryable(:tries => 12, :sleep => 5, :on => [::Aws::EC2::Errors::InvalidInstanceIDNotFound]) do
+        instance.create_tags({tags: [{key: "Name", value: machine_spec.name}]})
       end
-      current_tags = read_tags_block.call(aws_object)
-      # AWS always returns tags as strings, and we don't want to overwrite a
-      # tag-as-string with the same tag-as-symbol
-      desired_tags = Hash[desired_tags.map {|k, v| [k.to_s, v.to_s] }]
-      tags_to_update = desired_tags.reject {|k,v| current_tags[k] == v}
-      tags_to_delete = current_tags.keys - desired_tags.keys
-      # We don't want to delete `Name`, just all other tags
-      tags_to_delete.delete('Name')
-
-      unless tags_to_update.empty?
-        action_handler.perform_action "applying tags #{tags_to_update}" do
-          set_tags_block.call(aws_object, tags_to_update)
-        end
+      if machine_options.has_key?(:source_dest_check)
+        instance.modify_attribute({
+          source_dest_check: {
+            value: machine_options[:source_dest_check]
+          }
+        })
       end
-      unless tags_to_delete.empty?
-        action_handler.perform_action "deleting tags #{tags_to_delete.inspect}" do
-          delete_tags_block.call(aws_object, tags_to_delete)
+      machine_spec.reference = {
+          'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
+          'allocated_at' => Time.now.utc.to_s,
+          'host_node' => action_handler.host_node,
+          'image_id' => bootstrap_options[:image_id],
+          'instance_id' => instance.id
+      }
+      machine_spec.driver_url = driver_url
+      machine_spec.reference['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
+      # TODO 2.0 We no longer support `use_private_ip_for_ssh`, only `transport_address_location`
+      if machine_options[:use_private_ip_for_ssh]
+        unless @transport_address_location_warned
+          Chef::Log.warn("The machine_option ':use_private_ip_for_ssh' has been deprecated, use ':transport_address_location'")
+          @transport_address_location_warned = true
         end
+        machine_options = Cheffish::MergedConfig.new(machine_options, {:transport_address_location => :private_ip})
       end
-    end
-
-    def wait_until_taggable(instance)
-      Retryable.retryable(:tries => 12, :sleep => 5, :on => [AWS::EC2::Errors::InvalidInstanceID::NotFound, TimeoutError]) do
-        raise TimeoutError unless instance.status == :pending || instance.status == :running
+      %w(is_windows ssh_username sudo transport_address_location ssh_gateway).each do |key|
+        machine_spec.reference[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
       end
+      instance
     end
 
     def get_listeners(listeners)