chef-provisioning-aws 0.4.0 → 0.5.0

data/lib/chef/provisioning/aws_driver/driver.rb

@@ -47,10 +47,11 @@ module AWSDriver
       region = nil if region && region.empty?
 
       credentials = profile_name ? aws_credentials[profile_name] : aws_credentials.default
-      @aws_config = AWS::Core::Configuration.new(
+      @aws_config = AWS.config(
         access_key_id:     credentials[:aws_access_key_id],
         secret_access_key: credentials[:aws_secret_access_key],
-        region: region || credentials[:region]
+        region: region || credentials[:region],
+        logger: Chef::Log.logger
       )
     end
 
@@ -60,50 +61,30 @@ module AWSDriver
 
     # Load balancer methods
     def allocate_load_balancer(action_handler, lb_spec, lb_options, machine_specs)
-      lb_options ||= {}
-      if lb_options[:security_group_ids]
-        security_groups = ec2.security_groups.filter('group-id', lb_options[:security_group_ids]).to_a
-      elsif lb_options[:security_group_names]
-        security_groups = ec2.security_groups.filter('group-name', lb_options[:security_group_names]).to_a
-      else
-        security_groups = []
-      end
-      security_group_ids = security_groups.map { |sg| sg.id }
-
-      availability_zones = lb_options[:availability_zones] || []
-      subnets = lb_options[:subnets] || []
-      listeners = lb_options[:listeners]
-      scheme = lb_options[:scheme]
-
-      validate_listeners(listeners)
-      if !availability_zones.empty? && !subnets.empty?
-        raise "You cannot specify both `availability_zones` and `subnets`"
-      end
-
-      lb_optionals = {}
-      lb_optionals[:security_groups] = security_group_ids unless security_group_ids.empty?
-      lb_optionals[:availability_zones] = availability_zones unless availability_zones.empty?
-      lb_optionals[:subnets] = subnets unless subnets.empty?
-      lb_optionals[:listeners] = listeners if listeners
-      lb_optionals[:scheme] = scheme if scheme
+      lb_options = AWSResource.lookup_options(lb_options || {}, managed_entry_store: lb_spec.managed_entry_store, driver: self)
 
       old_elb = nil
       actual_elb = load_balancer_for(lb_spec)
-      if !actual_elb.exists?
+      if !actual_elb || !actual_elb.exists?
+        lb_options[:listeners] ||= get_listeners(:http)
+        if !lb_options[:subnets] && !lb_options[:availability_zones] && machine_specs
+          lb_options[:subnets] = machine_specs.map { |s| ec2.instances[s.reference['instance_id']].subnet }.uniq
+        end
+
         perform_action = proc { |desc, &block| action_handler.perform_action(desc, &block) }
+        Chef::Log.debug "AWS Load Balancer options: #{lb_options.inspect}"
 
-        security_group_names = security_groups.map { |sg| sg.name }.join(",")
+        updates = [ "create load balancer #{lb_spec.name} in #{aws_config.region}" ]
+        updates << "  enable availability zones #{lb_options[:availability_zones]}" if lb_options[:availability_zones]
+        updates << "  attach subnets #{lb_options[:subnets].join(', ')}" if lb_options[:subnets]
+        updates << "  with listeners #{lb_options[:listeners]}" if lb_options[:listeners]
+        updates << "  with security groups #{lb_options[:security_groups]}" if lb_options[:security_groups]
 
-        updates = [ "Create load balancer #{lb_spec.name} in #{aws_config.region}" ]
-        updates << "  enable availability zones #{availability_zones.join(', ')}" if availability_zones.size > 0
-        updates << "  attach subnets #{subnets.join(', ')}" if subnets.size > 0
-        updates << "  with listeners #{listeners.join(', ')}" if listeners && listeners.size > 0
-        updates << "  with security groups #{security_group_names}" if security_group_names
 
         action_handler.perform_action updates do
-          actual_elb = elb.load_balancers.create(lb_spec.name, lb_optionals)
+          actual_elb = elb.load_balancers.create(lb_spec.name, lb_options)
 
-          lb_spec.location = {
+          lb_spec.reference = {
             'driver_url' => driver_url,
             'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
             'allocated_at' => Time.now.utc.to_s,
@@ -117,24 +98,28 @@ module AWSDriver
         end
 
         # TODO: refactor this whole giant method into many smaller method calls
+        # TODO if we update scheme, we don't need to run any of the other updates.
+        # Also, if things aren't specified (such as machines / listeners), we
+        # need to grab them from the actual load balancer so we don't lose them.
+        # i.e. load_balancer 'blah' do
+        #   lb_options: { scheme: 'other_scheme' }
+        # end
+        # TODO we will leak the actual_elb if we fail to finish creating it
         # Update scheme - scheme is immutable once set, so if it is changing we need to delete the old
         # ELB and create a new one
-        if scheme && scheme.downcase != actual_elb.scheme
-          desc = ["  updating scheme to #{scheme}"]
+        if lb_options[:scheme] && lb_options[:scheme].downcase != actual_elb.scheme
+          desc = ["  updating scheme to #{lb_options[:scheme]}"]
           desc << "  WARN: scheme is immutable, so deleting and re-creating the ELB"
           perform_action.call(desc) do
             old_elb = actual_elb
-            actual_elb = elb.load_balancers.create(lb_spec.name, lb_optionals)
+            actual_elb = elb.load_balancers.create(lb_spec.name, lb_options)
           end
         end
 
         # Update security groups
-        if security_group_ids.empty?
-            Chef::Log.debug("No Security Groups specified. Load_balancer[#{actual_elb.name}] cannot have " +
-            "empty Security Groups, so assuming it only currently has the default Security Group.  No action taken.")
-        else
+        if lb_options[:security_groups]
           current = actual_elb.security_group_ids
-          desired = security_group_ids
+          desired = lb_options[:security_groups]
           if current != desired
             perform_action.call("  updating security groups to #{desired.to_a}") do
               elb.client.apply_security_groups_to_load_balancer(
@@ -145,148 +130,155 @@ module AWSDriver
           end
         end
 
-        # A subnet always belongs to an availability zone.  When specifying a ELB spec, you can either
-        # specify subnets OR AZs but not both.  You cannot specify multiple subnets in the same AZ.
-        # You must specify at least 1 subnet or AZ.  On an update you cannot remove all subnets
-        # or AZs - it must belong to one.
-        if !availability_zones.empty? && !subnets.empty?
-          # We do this check here because there is no atomic call we can make to specify both
-          # subnets and AZs at the same time
-          raise "You cannot specify both `availability_zones` and `subnets`"
-        end
-        # Users can switch from availability zones to subnets or vice versa.  To ensure we do not
-        # unassign all (which causes an AWS error) we first add all available ones, then remove
-        # an unecessary ones
-        actual_zones_subnets = {}
-        actual_elb.subnets.each do |subnet|
-          actual_zones_subnets[subnet.id] = subnet.availability_zone.name
-        end
-
-        # Only 1 of subnet or AZ will be populated b/c of our check earlier
-        desired_subnets_zones = {}
-        availability_zones.each do |zone|
-          # If the user specifies availability zone, we find the default subnet for that
-          # AZ because this duplicates the create logic
-          zone = zone.downcase
-          filters = [
-            {:name => 'availabilityZone', :values => [zone]},
-            {:name => 'defaultForAz', :values => ['true']}
-          ]
-          default_subnet = ec2.client.describe_subnets(:filters => filters)[:subnet_set]
-          if default_subnet.size != 1
-            raise "Could not find default subnet in availability zone #{zone}"
+        if lb_options[:availability_zones] || lb_options[:subnets]
+          # A subnet always belongs to an availability zone.  When specifying a ELB spec, you can either
+          # specify subnets OR AZs but not both.  You cannot specify multiple subnets in the same AZ.
+          # You must specify at least 1 subnet or AZ.  On an update you cannot remove all subnets
+          # or AZs - it must belong to one.
+          if lb_options[:availability_zones] && lb_options[:subnets]
+            # We do this check here because there is no atomic call we can make to specify both
+            # subnets and AZs at the same time
+            raise "You cannot specify both `availability_zones` and `subnets`"
           end
-          default_subnet = default_subnet[0]
-          desired_subnets_zones[default_subnet[:subnet_id]] = zone
-        end
-        unless subnets.empty?
-          subnet_query = ec2.client.describe_subnets(:subnet_ids => subnets)[:subnet_set]
-          # AWS raises an error on an unknown subnet, but not an unknown AZ
-          subnet_query.each do |subnet|
-            zone = subnet[:availability_zone].downcase
-            desired_subnets_zones[subnet[:subnet_id]] = zone
+
+          # Users can switch from availability zones to subnets or vice versa.  To ensure we do not
+          # unassign all (which causes an AWS error) we first add all available ones, then remove
+          # an unecessary ones
+          actual_zones_subnets = {}
+          actual_elb.subnets.each do |subnet|
+            actual_zones_subnets[subnet.id] = subnet.availability_zone.name
           end
-        end
 
-        # We only bother attaching subnets, because doing this automatically attaches the AZ
-        attach_subnets = desired_subnets_zones.keys - actual_zones_subnets.keys
-        unless attach_subnets.empty?
-          action = "  attach subnets #{attach_subnets.join(', ')}"
-          enable_zones = (desired_subnets_zones.map {|s,z| z if attach_subnets.include?(s)}).compact
-          action += " (availability zones #{enable_zones.join(', ')})"
-          perform_action.call(action) do
-            begin
-              elb.client.attach_load_balancer_to_subnets(
-                load_balancer_name: actual_elb.name,
-                subnets: attach_subnets
-              )
-            rescue AWS::ELB::Errors::InvalidConfigurationRequest
-              raise "You cannot currently move from 1 subnet to another in the same availability zone. " +
-                  "Amazon does not have an atomic operation which allows this.  You must create a new " +
-                  "ELB with the correct subnets and move instances into it.  Tried to attach subets " +
-                  "#{attach_subnets.join(', ')} (availability zones #{enable_zones.join(', ')}) to " +
-                  "existing ELB named #{actual_elb.name}"
+          # Only 1 of subnet or AZ will be populated b/c of our check earlier
+          desired_subnets_zones = {}
+          if lb_options[:availability_zones]
+            lb_options[:availability_zones].each do |zone|
+              # If the user specifies availability zone, we find the default subnet for that
+              # AZ because this duplicates the create logic
+              zone = zone.downcase
+              filters = [
+                {:name => 'availabilityZone', :values => [zone]},
+                {:name => 'defaultForAz', :values => ['true']}
+              ]
+              default_subnet = ec2.client.describe_subnets(:filters => filters)[:subnet_set]
+              if default_subnet.size != 1
+                raise "Could not find default subnet in availability zone #{zone}"
+              end
+              default_subnet = default_subnet[0]
+              desired_subnets_zones[default_subnet[:subnet_id]] = zone
+            end
+          end
+          unless lb_options[:subnets] && lb_options[:subnets.empty?]
+            subnet_query = ec2.client.describe_subnets(:subnet_ids => lb_options[:subnets])[:subnet_set]
+            # AWS raises an error on an unknown subnet, but not an unknown AZ
+            subnet_query.each do |subnet|
+              zone = subnet[:availability_zone].downcase
+              desired_subnets_zones[subnet[:subnet_id]] = zone
+            end
+          end
+
+          # We only bother attaching subnets, because doing this automatically attaches the AZ
+          attach_subnets = desired_subnets_zones.keys - actual_zones_subnets.keys
+          unless attach_subnets.empty?
+            action = "  attach subnets #{attach_subnets.join(', ')}"
+            enable_zones = (desired_subnets_zones.map {|s,z| z if attach_subnets.include?(s)}).compact
+            action += " (availability zones #{enable_zones.join(', ')})"
+            perform_action.call(action) do
+              begin
+                elb.client.attach_load_balancer_to_subnets(
+                  load_balancer_name: actual_elb.name,
+                  subnets: attach_subnets
+                )
+              rescue AWS::ELB::Errors::InvalidConfigurationRequest
+                raise "You cannot currently move from 1 subnet to another in the same availability zone. " +
+                    "Amazon does not have an atomic operation which allows this.  You must create a new " +
+                    "ELB with the correct subnets and move instances into it.  Tried to attach subets " +
+                    "#{attach_subnets.join(', ')} (availability zones #{enable_zones.join(', ')}) to " +
+                    "existing ELB named #{actual_elb.name}"
+              end
             end
           end
-        end
 
-        detach_subnets = actual_zones_subnets.keys - desired_subnets_zones.keys
-        unless detach_subnets.empty?
-          action = "  detach subnets #{detach_subnets.join(', ')}"
-          disable_zones = (actual_zones_subnets.map {|s,z| z if detach_subnets.include?(s)}).compact
-          action += " (availability zones #{disable_zones.join(', ')})"
-          perform_action.call(action) do
-            elb.client.detach_load_balancer_from_subnets(
-              load_balancer_name: actual_elb.name,
-              subnets: detach_subnets
-            )
+          detach_subnets = actual_zones_subnets.keys - desired_subnets_zones.keys
+          unless detach_subnets.empty?
+            action = "  detach subnets #{detach_subnets.join(', ')}"
+            disable_zones = (actual_zones_subnets.map {|s,z| z if detach_subnets.include?(s)}).compact
+            action += " (availability zones #{disable_zones.join(', ')})"
+            perform_action.call(action) do
+              elb.client.detach_load_balancer_from_subnets(
+                load_balancer_name: actual_elb.name,
+                subnets: detach_subnets
+              )
+            end
           end
         end
 
         # Update listeners - THIS IS NOT ATOMIC
-        add_listeners = {}
-        listeners.each { |l| add_listeners[l[:port]] = l } if listeners
-        actual_elb.listeners.each do |listener|
-          desired_listener = add_listeners.delete(listener.port)
-          if desired_listener
-
-            # listener.(port|protocol|instance_port|instance_protocol) are immutable for the life
-            # of the listener - must create a new one and delete old one
-            immutable_updates = []
-            if listener.protocol != desired_listener[:protocol].to_sym.downcase
-              immutable_updates << "    update protocol from #{listener.protocol.inspect} to #{desired_listener[:protocol].inspect}"
-            end
-            if listener.instance_port != desired_listener[:instance_port]
-              immutable_updates << "    update instance port from #{listener.instance_port.inspect} to #{desired_listener[:instance_port].inspect}"
-            end
-            if listener.instance_protocol != desired_listener[:instance_protocol].to_sym.downcase
-              immutable_updates << "    update instance protocol from #{listener.instance_protocol.inspect} to #{desired_listener[:instance_protocol].inspect}"
-            end
-            if !immutable_updates.empty?
-              perform_action.call(immutable_updates) do
-                listener.delete
-                actual_elb.listeners.create(desired_listener)
+        if lb_options[:listeners]
+          add_listeners = {}
+          lb_options[:listeners].each { |l| add_listeners[l[:port]] = l }
+          actual_elb.listeners.each do |listener|
+            desired_listener = add_listeners.delete(listener.port)
+            if desired_listener
+
+              # listener.(port|protocol|instance_port|instance_protocol) are immutable for the life
+              # of the listener - must create a new one and delete old one
+              immutable_updates = []
+              if listener.protocol != desired_listener[:protocol].to_sym.downcase
+                immutable_updates << "    update protocol from #{listener.protocol.inspect} to #{desired_listener[:protocol].inspect}"
               end
-            elsif listener.server_certificate != desired_listener[:server_certificate]
-              # Server certificate is mutable - if no immutable changes required a full recreate, update cert
-              perform_action.call("    update server certificate from #{listener.server_certificate} to #{desired_listener[:server_certificate]}") do
-                listener.server_certificate = desired_listener[:server_certificate]
+              if listener.instance_port != desired_listener[:instance_port]
+                immutable_updates << "    update instance port from #{listener.instance_port.inspect} to #{desired_listener[:instance_port].inspect}"
+              end
+              if listener.instance_protocol != desired_listener[:instance_protocol].to_sym.downcase
+                immutable_updates << "    update instance protocol from #{listener.instance_protocol.inspect} to #{desired_listener[:instance_protocol].inspect}"
+              end
+              if !immutable_updates.empty?
+                perform_action.call(immutable_updates) do
+                  listener.delete
+                  actual_elb.listeners.create(desired_listener)
+                end
+              elsif listener.server_certificate != desired_listener[:server_certificate]
+                # Server certificate is mutable - if no immutable changes required a full recreate, update cert
+                perform_action.call("    update server certificate from #{listener.server_certificate} to #{desired_listener[:server_certificate]}") do
+                  listener.server_certificate = desired_listener[:server_certificate]
+                end
               end
-            end
 
-          else
-            perform_action.call("  remove listener #{listener.port}") do
-              listener.delete
+            else
+              perform_action.call("  remove listener #{listener.port}") do
+                listener.delete
+              end
             end
           end
-        end
-        add_listeners.values.each do |listener|
-          updates = [ "  add listener #{listener[:port]}" ]
-          updates << "    set protocol to #{listener[:protocol].inspect}"
-          updates << "    set instance port to #{listener[:instance_port].inspect}"
-          updates << "    set instance protocol to #{listener[:instance_protocol].inspect}"
-          updates << "    set server certificate to #{listener[:server_certificate]}" if listener[:server_certificate]
-          perform_action.call(updates) do
-            actual_elb.listeners.create(listener)
+          add_listeners.values.each do |listener|
+            updates = [ "  add listener #{listener[:port]}" ]
+            updates << "    set protocol to #{listener[:protocol].inspect}"
+            updates << "    set instance port to #{listener[:instance_port].inspect}"
+            updates << "    set instance protocol to #{listener[:instance_protocol].inspect}"
+            updates << "    set server certificate to #{listener[:server_certificate]}" if listener[:server_certificate]
+            perform_action.call(updates) do
+              actual_elb.listeners.create(listener)
+            end
           end
         end
       end
 
       # Update instance list, but only if there are machines specified
-      actual_instance_ids = actual_elb.instances.map { |i| i.instance_id }
-
       if machine_specs
-        instances_to_add = machine_specs.select { |s| !actual_instance_ids.include?(s.location['instance_id']) }
-        instance_ids_to_remove = actual_instance_ids - machine_specs.map { |s| s.location['instance_id'] }
-  
+        actual_instance_ids = actual_elb.instances.map { |i| i.instance_id }
+
+        instances_to_add = machine_specs.select { |s| !actual_instance_ids.include?(s.reference['instance_id']) }
+        instance_ids_to_remove = actual_instance_ids - machine_specs.map { |s| s.reference['instance_id'] }
+
         if instances_to_add.size > 0
           perform_action.call("  add machines #{instances_to_add.map { |s| s.name }.join(', ')}") do
-            instance_ids_to_add = instances_to_add.map { |s| s.location['instance_id'] }
+            instance_ids_to_add = instances_to_add.map { |s| s.reference['instance_id'] }
             Chef::Log.debug("Adding instances #{instance_ids_to_add.join(', ')} to load balancer #{actual_elb.name} in region #{aws_config.region}")
             actual_elb.instances.add(instance_ids_to_add)
           end
         end
-  
+
         if instance_ids_to_remove.size > 0
           perform_action.call("  remove instances #{instance_ids_to_remove}") do
             actual_elb.instances.remove(instance_ids_to_remove)
@@ -303,12 +295,12 @@ module AWSDriver
       # Something went wrong before we could moved instances from the old ELB to the new one
       # Don't delete the old ELB, but warn users there could now be 2 ELBs with the same name
       unless old_elb.nil?
-        Chef::Log.warn("It is possible there are now 2 ELB instances - #{old_elb.id} and #{actual_elb.id}. " +
+        Chef::Log.warn("It is possible there are now 2 ELB instances - #{old_elb.name} and #{actual_elb.name}. " +
         "Determine which is correct and manually clean up the other.")
       end
     end
 
-    def ready_load_balancer(action_handler, lb_spec, lb_options, machine_specs)
+    def ready_load_balancer(action_handler, lb_spec, lb_options, machine_spec)
     end
 
     def destroy_load_balancer(action_handler, lb_spec, lb_options)
@@ -327,27 +319,21 @@ module AWSDriver
     end
 
     # Image methods
-    def allocate_image(action_handler, image_spec, image_options, machine_spec)
+    def allocate_image(action_handler, image_spec, image_options, machine_spec, machine_options)
       actual_image = image_for(image_spec)
       if actual_image.nil? || !actual_image.exists? || actual_image.state == :failed
         action_handler.perform_action "Create image #{image_spec.name} from machine #{machine_spec.name} with options #{image_options.inspect}" do
           image_options[:name] ||= image_spec.name
-          image_options[:instance_id] ||= machine_spec.location['instance_id']
+          image_options[:instance_id] ||= machine_spec.reference['instance_id']
           image_options[:description] ||= "Image #{image_spec.name} created from machine #{machine_spec.name}"
           Chef::Log.debug "AWS Image options: #{image_options.inspect}"
           image = ec2.images.create(image_options.to_hash)
-          image_spec.location = {
+          image_spec.reference = {
             'driver_url' => driver_url,
             'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
             'image_id' => image.id,
             'allocated_at' => Time.now.to_i
           }
-          image_spec.machine_options ||= {}
-          image_spec.machine_options.merge!({
-            :bootstrap_options => {
-                :image_id => image.id
-            }
-          })
         end
       end
     end
@@ -368,10 +354,12 @@ module AWSDriver
 
     def destroy_image(action_handler, image_spec, image_options)
       actual_image = image_for(image_spec)
-      snapshots = snapshots_for(image_spec)
       if actual_image.nil? || !actual_image.exists?
         Chef::Log.warn "Image #{image_spec.name} doesn't exist"
       else
+        snapshots = actual_image.block_device_mappings.map do |dev, opts|
+          ec2.snapshots[opts[:snapshot_id]]
+        end
         action_handler.perform_action "De-registering image #{image_spec.name}" do
           actual_image.deregister
         end
@@ -420,7 +408,7 @@ EOD
           sleep 5 while instance.status == :pending
           # TODO add other tags identifying user / node url (same as fog)
           instance.tags['Name'] = machine_spec.name
-          machine_spec.location = {
+          machine_spec.reference = {
               'driver_url' => driver_url,
               'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
               'allocated_at' => Time.now.utc.to_s,
@@ -428,9 +416,9 @@ EOD
               'image_id' => bootstrap_options[:image_id],
               'instance_id' => instance.id
           }
-          machine_spec.location['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
+          machine_spec.reference['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
           %w(is_windows ssh_username sudo use_private_ip_for_ssh ssh_gateway).each do |key|
-            machine_spec.location[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
+            machine_spec.reference[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
           end
         end
       end
@@ -455,7 +443,7 @@ EOD
       if instance.status != :running
         wait_until_machine(action_handler, machine_spec, instance) { instance.status != :stopping }
         if instance.status == :stopped
-          action_handler.perform_action "Start #{machine_spec.name} (#{machine_spec.location['instance_id']}) in #{aws_config.region} ..." do
+          action_handler.perform_action "Start #{machine_spec.name} (#{machine_spec.reference['instance_id']}) in #{aws_config.region} ..." do
             instance.start
           end
         end
@@ -473,19 +461,19 @@ EOD
         machine_spec = Chef::Provisioning::ChefMachineSpec.get(name, chef_server)
       end
 
-      machine_for(machine_spec, machine_spec.location)
+      machine_for(machine_spec, machine_spec.reference)
     end
 
     def destroy_machine(action_handler, machine_spec, machine_options)
       instance = instance_for(machine_spec)
       if instance && instance.exists?
         # TODO do we need to wait_until(action_handler, machine_spec, instance) { instance.status != :shutting_down } ?
-        action_handler.perform_action "Terminate #{machine_spec.name} (#{machine_spec.location['instance_id']}) in #{aws_config.region} ..." do
+        action_handler.perform_action "Terminate #{machine_spec.name} (#{machine_spec.reference['instance_id']}) in #{aws_config.region} ..." do
           instance.terminate
-          machine_spec.location = nil
+          machine_spec.reference = nil
         end
       else
-        Chef::Log.warn "Instance #{machine_spec.location['instance_id']} doesn't exist for #{machine_spec.name}"
+        Chef::Log.warn "Instance #{machine_spec.reference['instance_id']} doesn't exist for #{machine_spec.name}"
       end
 
       strategy = convergence_strategy_for(machine_spec, machine_options)
@@ -500,6 +488,14 @@ EOD
       @elb ||= AWS::ELB.new(config: aws_config)
     end
 
+    def iam
+      @iam ||= AWS::IAM.new(config: aws_config)
+    end
+
+    def s3
+      @s3 ||= AWS::S3.new(config: aws_config)
+    end
+
     def sns
       @sns ||= AWS::SNS.new(config: aws_config)
     end
@@ -508,15 +504,40 @@ EOD
       @sqs ||= AWS::SQS.new(config: aws_config)
     end
 
-    def s3
-      @s3 ||= AWS::S3.new(config: aws_config)
-    end
-
     def auto_scaling
       @auto_scaling ||= AWS::AutoScaling.new(config: aws_config)
     end
 
-    private
+    def build_arn(partition: 'aws', service: nil, region: aws_config.region, account_id: self.account_id, resource: nil)
+      "arn:#{partition}:#{service}:#{region}:#{account_id}:#{resource}"
+    end
+
+    def parse_arn(arn)
+      parts = arn.split(':', 6)
+      {
+        partition: parts[1],
+        service: parts[2],
+        region: parts[3],
+        account_id: parts[4],
+        resource: parts[5]
+      }
+    end
+
+    def account_id
+      begin
+        # We've got an AWS account root credential or an IAM admin with access rights
+        current_user = iam.client.get_user
+        arn = current_user[:user][:arn]
+      rescue AWS::IAM::Errors::AccessDenied => e
+        # If we don't have access, the error message still tells us our account ID and user ...
+        # https://forums.aws.amazon.com/thread.jspa?messageID=394344
+        if e.to_s !~ /\b(arn:aws:iam::[0-9]{12}:\S*)/
+          raise "IAM error response for GetUser did not include user ARN.  Can't retrieve account ID."
+        end
+        arn = $1
+      end
+      parse_arn(arn)[:account_id]
+    end
 
     # For creating things like AWS keypairs exclusively
     @@chef_default_lock = Mutex.new
@@ -528,7 +549,7 @@ EOD
         raise "Instance for node #{machine_spec.name} has not been created!"
       end
 
-      if machine_spec.location['is_windows']
+      if machine_spec.reference['is_windows']
         Chef::Provisioning::Machine::WindowsMachine.new(machine_spec, transport_for(machine_spec, machine_options, instance), convergence_strategy_for(machine_spec, machine_options))
       else
         Chef::Provisioning::Machine::UnixMachine.new(machine_spec, transport_for(machine_spec, machine_options, instance), convergence_strategy_for(machine_spec, machine_options))
@@ -551,6 +572,7 @@ EOD
         Chef::Log.debug "Non-windows, not setting userdata"
       end
 
+      bootstrap_options = AWSResource.lookup_options(bootstrap_options, managed_entry_store: machine_spec.managed_entry_store, driver: self)
       Chef::Log.debug "AWS Bootstrap options: #{bootstrap_options.inspect}"
       bootstrap_options
     end
@@ -571,62 +593,36 @@ EOD
     end
 
     def load_balancer_for(lb_spec)
-      if lb_spec.name
-        elb.load_balancers[lb_spec.name]
-      else
-        nil
-      end
+      Chef::Resource::AwsLoadBalancer.get_aws_object(lb_spec.name, driver: self, managed_entry_store: lb_spec.managed_entry_store, required: false)
     end
 
     def instance_for(machine_spec)
-      if machine_spec.location && machine_spec.location['instance_id']
-        ec2.instances[machine_spec.location['instance_id']]
+      if machine_spec.reference
+        if machine_spec.reference['driver_url'] != driver_url
+          raise "Switching a machine's driver from #{machine_spec.reference['driver_url']} to #{driver_url} is not currently supported!  Use machine :destroy and then re-create the machine on the new driver."
+        end
+        Chef::Resource::AwsInstance.get_aws_object(machine_spec.reference['instance_id'], driver: self, managed_entry_store: machine_spec.managed_entry_store, required: false)
       end
     end
 
     def instances_for(machine_specs)
       result = {}
-      machine_specs.each do |machine_spec|
-        if machine_spec.location && machine_spec.location['instance_id']
-          if machine_spec.location['driver_url'] != driver_url
-            raise "Switching a machine's driver from #{machine_spec.location['driver_url']} to #{driver_url} is not currently supported!  Use machine :destroy and then re-create the machine on the new driver."
-          end
-          #returns nil if not found
-          result[machine_spec] = ec2.instances[machine_spec.location['instance_id']]
-        end
-      end
+      machine_specs.each { |machine_spec| result[machine_spec] = instance_for(machine_spec) }
       result
     end
 
     def image_for(image_spec)
-      if image_spec.location && image_spec.location['image_id']
-        ec2.images[image_spec.location['image_id']]
-      end
-    end
-
-    def snapshots_for(image_spec)
-      if image_spec.location && image_spec.location['image_id']
-        actual_image = image_for(image_spec)
-        snapshots = []
-        actual_image.block_device_mappings.each do |dev, opts|
-            snapshots << ec2.snapshots[opts[:snapshot_id]]
-        end
-        snapshots
-      end
+      Chef::Resource::AwsImage.get_aws_object(image_spec.name, driver: self, managed_entry_store: image_spec.managed_entry_store, required: false)
     end
 
     def transport_for(machine_spec, machine_options, instance)
-      if machine_spec.location['is_windows']
+      if machine_spec.reference['is_windows']
         create_winrm_transport(machine_spec, machine_options, instance)
       else
         create_ssh_transport(machine_spec, machine_options, instance)
       end
     end
 
-    def compute_options
-
-    end
-
     def aws_credentials
       # Grab the list of possible credentials
       @aws_credentials ||= if driver_options[:aws_credentials]
@@ -674,7 +670,7 @@ EOD
     def create_winrm_transport(machine_spec, machine_options, instance)
       remote_host = determine_remote_host(machine_spec, instance)
 
-      port = machine_spec.location['winrm_port'] || 5985
+      port = machine_spec.reference['winrm_port'] || 5985
       endpoint = "http://#{remote_host}:#{port}/wsman"
       type = :plaintext
       pem_bytes = get_private_key(instance.key_name)
@@ -685,7 +681,7 @@ EOD
       decrypted_password = private_key.private_decrypt decoded
 
       winrm_options = {
-        :user => machine_spec.location['winrm_username'] || 'Administrator',
+        :user => machine_spec.reference['winrm_username'] || 'Administrator',
         :pass => decrypted_password,
         :disable_sspi => true,
         :basic_auth_only => true
@@ -699,7 +695,7 @@ EOD
       sleep_time = 10
       max_wait_time = 900 # 15 minutes
       encrypted_admin_password = nil
-      instance_id = machine_spec.location['instance_id']
+      instance_id = machine_spec.reference['instance_id']
 
       Chef::Log.info "waiting for #{machine_spec.name}'s admin password to be available..."
       while time_elapsed < max_wait_time && encrypted_admin_password.nil?
@@ -720,12 +716,12 @@ EOD
 
     def create_ssh_transport(machine_spec, machine_options, instance)
       ssh_options = ssh_options_for(machine_spec, machine_options, instance)
-      username = machine_spec.location['ssh_username'] || machine_options[:ssh_username] || default_ssh_username
-      if machine_options.has_key?(:ssh_username) && machine_options[:ssh_username] != machine_spec.location['ssh_username']
-        Chef::Log.warn("Server #{machine_spec.name} was created with SSH username #{machine_spec.location['ssh_username']} and machine_options specifies username #{machine_options[:ssh_username]}.  Using #{machine_spec.location['ssh_username']}.  Please edit the node and change the chef_provisioning.location.ssh_username attribute if you want to change it.")
+      username = machine_spec.reference['ssh_username'] || machine_options[:ssh_username] || default_ssh_username
+      if machine_options.has_key?(:ssh_username) && machine_options[:ssh_username] != machine_spec.reference['ssh_username']
+        Chef::Log.warn("Server #{machine_spec.name} was created with SSH username #{machine_spec.reference['ssh_username']} and machine_options specifies username #{machine_options[:ssh_username]}.  Using #{machine_spec.reference['ssh_username']}.  Please edit the node and change the chef_provisioning.reference.ssh_username attribute if you want to change it.")
       end
       options = {}
-      if machine_spec.location[:sudo] || (!machine_spec.location.has_key?(:sudo) && username != 'root')
+      if machine_spec.reference[:sudo] || (!machine_spec.reference.has_key?(:sudo) && username != 'root')
         options[:prefix] = 'sudo '
       end
 
@@ -733,13 +729,13 @@ EOD
 
       #Enable pty by default
       options[:ssh_pty_enable] = true
-      options[:ssh_gateway] = machine_spec.location['ssh_gateway'] if machine_spec.location.has_key?('ssh_gateway')
+      options[:ssh_gateway] = machine_spec.reference['ssh_gateway'] if machine_spec.reference.has_key?('ssh_gateway')
 
       Chef::Provisioning::Transport::SSH.new(remote_host, username, ssh_options, options, config)
     end
 
     def determine_remote_host(machine_spec, instance)
-      if machine_spec.location['use_private_ip_for_ssh']
+      if machine_spec.reference['use_private_ip_for_ssh']
         instance.private_ip_address
       elsif !instance.public_ip_address
         Chef::Log.warn("Server #{machine_spec.name} has no public ip address.  Using private ip '#{instance.private_ip_address}'.  Set driver option 'use_private_ip_for_ssh' => true if this will always be the case ...")
@@ -768,10 +764,10 @@ EOD
           raise "Server has key name '#{instance.key_name}', but the corresponding private key was not found locally.  Check if the key is in Chef::Config.private_key_paths: #{Chef::Config.private_key_paths.join(', ')}"
         end
         result[:key_data] = [ key ]
-      elsif machine_spec.location['key_name']
-        key = get_private_key(machine_spec.location['key_name'])
+      elsif machine_spec.reference['key_name']
+        key = get_private_key(machine_spec.reference['key_name'])
         unless key
-          raise "Server was created with key name '#{machine_spec.location['key_name']}', but the corresponding private key was not found locally.  Check if the key is in Chef::Config.private_key_paths: #{Chef::Config.private_key_paths.join(', ')}"
+          raise "Server was created with key name '#{machine_spec.reference['key_name']}', but the corresponding private key was not found locally.  Check if the key is in Chef::Config.private_key_paths: #{Chef::Config.private_key_paths.join(', ')}"
         end
         result[:key_data] = [ key ]
       elsif machine_options[:bootstrap_options] && machine_options[:bootstrap_options][:key_path]
@@ -780,7 +776,7 @@ EOD
         result[:key_data] = [ get_private_key(machine_options[:bootstrap_options][:key_name]) ]
       else
         # TODO make a way to suggest other keys to try ...
-        raise "No key found to connect to #{machine_spec.name} (#{machine_spec.location.inspect})!"
+        raise "No key found to connect to #{machine_spec.name} (#{machine_spec.reference.inspect})!"
       end
       result
     end
@@ -792,11 +788,11 @@ EOD
         ohai_hints: { 'ec2' => '' })
 
       # Defaults
-      if !machine_spec.location
+      if !machine_spec.reference
         return Chef::Provisioning::ConvergenceStrategy::NoConverge.new(convergence_options, config)
       end
 
-      if machine_spec.location['is_windows']
+      if machine_spec.reference['is_windows']
         Chef::Provisioning::ConvergenceStrategy::InstallMsi.new(convergence_options, config)
       elsif machine_options[:cached_installer] == true
         Chef::Provisioning::ConvergenceStrategy::InstallCached.new(convergence_options, config)
@@ -816,11 +812,14 @@ EOD
       max_wait_time = 120
       if !yield(image)
         action_handler.report_progress "waiting for #{image_spec.name} (#{image.id} on #{driver_url}) to be ready ..."
-        while time_elapsed < 120 && !yield(image)
+        while time_elapsed < max_wait_time && !yield(image)
           action_handler.report_progress "been waiting #{time_elapsed}/#{max_wait_time} -- sleeping #{sleep_time} seconds for #{image_spec.name} (#{image.id} on #{driver_url}) to be ready ..."
           sleep(sleep_time)
           time_elapsed += sleep_time
         end
+        unless yield(image)
+          raise "Image #{image.id} did not become ready within 120 seconds"
+        end
         action_handler.report_progress "Image #{image_spec.name} is now ready"
       end
     end
@@ -837,11 +836,14 @@ EOD
       if !yield(instance)
         if action_handler.should_perform_actions
           action_handler.report_progress "waiting for #{machine_spec.name} (#{instance.id} on #{driver_url}) to be ready ..."
-          while time_elapsed < 120 && !yield(instance)
+          while time_elapsed < max_wait_time && !yield(instance)
             action_handler.report_progress "been waiting #{time_elapsed}/#{max_wait_time} -- sleeping #{sleep_time} seconds for #{machine_spec.name} (#{instance.id} on #{driver_url}) to be ready ..."
             sleep(sleep_time)
             time_elapsed += sleep_time
           end
+          unless yield(instance)
+            raise "Image #{instance.id} did not become ready within 120 seconds"
+          end
           action_handler.report_progress "#{machine_spec.name} is now ready"
         end
       end
@@ -856,7 +858,7 @@ EOD
       unless transport.available?
         if action_handler.should_perform_actions
           action_handler.report_progress "waiting for #{machine_spec.name} (#{instance.id} on #{driver_url}) to be connectable (transport up and running) ..."
-          while time_elapsed < 120 && !transport.available?
+          while time_elapsed < max_wait_time && !transport.available?
             action_handler.report_progress "been waiting #{time_elapsed}/#{max_wait_time} -- sleeping #{sleep_time} seconds for #{machine_spec.name} (#{instance.id} on #{driver_url}) to be connectable ..."
             sleep(sleep_time)
             time_elapsed += sleep_time
@@ -868,8 +870,8 @@ EOD
     end
 
     def default_aws_keypair_name(machine_spec)
-      if machine_spec.location &&
-          Gem::Version.new(machine_spec.location['driver_version']) < Gem::Version.new('0.10')
+      if machine_spec.reference &&
+          Gem::Version.new(machine_spec.reference['driver_version']) < Gem::Version.new('0.10')
         'metal_default'
       else
         'chef_default'
@@ -908,8 +910,8 @@ EOD
             yield machine_spec, actual_instance if block_given?
             next
           end
-        elsif machine_spec.location
-          Chef::Log.warn "Machine #{machine_spec.name} (#{machine_spec.location['instance_id']} on #{driver_url}) no longer exists.  Recreating ..."
+        elsif machine_spec.reference
+          Chef::Log.warn "Machine #{machine_spec.name} (#{machine_spec.reference['instance_id']} on #{driver_url}) no longer exists.  Recreating ..."
         end
 
         bootstrap_options = bootstrap_options_for(action_handler, machine_spec, machine_options)
@@ -934,7 +936,7 @@ EOD
             # Assign each one to a machine spec
             machine_spec = machine_specs.pop
             machine_options = specs_and_options[machine_spec]
-            machine_spec.location = {
+            machine_spec.reference = {
               'driver_url' => driver_url,
               'driver_version' => Chef::Provisioning::AWSDriver::VERSION,
               'allocated_at' => Time.now.utc.to_s,
@@ -943,9 +945,9 @@ EOD
               'instance_id' => instance.id
             }
             instance.tags['Name'] = machine_spec.name
-            machine_spec.location['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
+            machine_spec.reference['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
             %w(is_windows ssh_username sudo use_private_ip_for_ssh ssh_gateway).each do |key|
-              machine_spec.location[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
+              machine_spec.reference[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
             end
             action_handler.performed_action "machine #{machine_spec.name} created as #{instance.id} on #{driver_url}"
 
@@ -969,19 +971,79 @@ EOD
       end.to_a
     end
 
-    # The listeners API is different between the SDK v1 and v2
-    # http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/ELB/Listener.html
-    VALID_LISTENER_KEYS = [:port, :protocol, :instance_port, :instance_protocol]
-    def validate_listeners(listeners)
-      listeners.each do |listener|
-        listener.keys.each do |k|
-          unless VALID_LISTENER_KEYS.include?(k)
-            raise "#{k} is an invalid listener key, can be one of #{VALID_LISTENER_KEYS.inspect}"
-          end
+    def get_listeners(listeners)
+      case listeners
+      when Hash
+        listeners.map do |from, to|
+          from = get_listener(from)
+          from.delete(:instance_port)
+          from.delete(:instance_protocol)
+          to = get_listener(to)
+          to.delete(:port)
+          to.delete(:protocol)
+          to.merge(from)
         end
+      when Array
+        listeners.map { |listener| get_listener(listener) }
+      when nil
+        nil
+      else
+        [ get_listener(listeners) ]
       end
     end
 
+    def get_listener(listener)
+      result = {}
+
+      case listener
+      when Hash
+        result.merge!(listener)
+      when Array
+        result[:port] = listener[0] if listener.size >= 1
+        result[:protocol] = listener[1] if listener.size >= 2
+      when Symbol,String
+        result[:protocol] = listener
+      when Integer
+        result[:port] = listener
+      else
+        raise "Invalid listener #{listener}"
+      end
+
+      # If either port or protocol are set, set the other
+      if result[:port] && !result[:protocol]
+        result[:protocol] = PROTOCOL_DEFAULTS[result[:port]]
+      elsif result[:protocol] && !result[:port]
+        result[:port] = PORT_DEFAULTS[result[:protocol]]
+      end
+      if result[:instance_port] && !result[:instance_protocol]
+        result[:instance_protocol] = PROTOCOL_DEFAULTS[result[:instance_port]]
+      elsif result[:instance_protocol] && !result[:instance_port]
+        result[:instance_port] = PORT_DEFAULTS[result[:instance_protocol]]
+      end
+
+      # If instance_port is still unset, copy port/protocol over
+      result[:instance_port] ||= result[:port]
+      result[:instance_protocol] ||= result[:protocol]
+
+      result
+    end
+
+    def default_instance_type
+      't1.micro'
+    end
+
+    PORT_DEFAULTS = {
+      :http => 80,
+      :https => 443,
+    }
+    PROTOCOL_DEFAULTS = {
+      25 => :tcp,
+      80 => :http,
+      443 => :https,
+      465 => :ssl,
+      587 => :tcp,
+    }
+
   end
 end
 end