cloud-mu 3.3.0 → 3.5.1

data/modules/mu/providers/aws/search_domain.rb

@@ -36,7 +36,7 @@ module MU
 
           MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params
           @cloud_id = @config['domain_name']
-          MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).create_elasticsearch_domain(params).domain_status
+          MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).create_elasticsearch_domain(params).domain_status
 
           tagDomain
 
@@ -52,7 +52,7 @@ module MU
             waitWhileProcessing # wait until the create finishes, if still going
 
             MU.log "Updating ElasticSearch domain #{@config['domain_name']}", MU::NOTICE, details: params
-            MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).update_elasticsearch_domain_config(params)
+            MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).update_elasticsearch_domain_config(params)
           end
 
           waitWhileProcessing # don't return until creation/updating is complete
@@ -68,7 +68,7 @@ module MU
           @cloud_id ||= @config['domain_name']
           return nil if !@cloud_id
           MU.retrier([::Aws::ElasticsearchService::Errors::ResourceNotFoundException], wait: 10, max: 12) {
-            @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain(
+            @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).describe_elasticsearch_domain(
               domain_name: @cloud_id
             ).domain_status
           }
@@ -88,7 +88,7 @@ module MU
         def notify
           return nil if !cloud_desc(use_cache: false)
           deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true)
-          tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: arn).tag_list
+          tags = MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).list_tags(arn: arn).tag_list
           deploy_struct['tags'] = tags.map { |t| { t.key => t.value } }
           if deploy_struct['endpoint']
             deploy_struct['kibana'] = deploy_struct['endpoint']+"/_plugin/kibana/"
@@ -200,7 +200,7 @@ module MU
             "cloud" => "AWS",
             "credentials" => @credentials,
             "cloud_id" => @cloud_id,
-            "region" => @config['region']
+            "region" => @region
           }
 
           if !cloud_desc
@@ -241,7 +241,7 @@ module MU
             bok['identity_pool_id'] = cloud_desc.cognito_options.identity_pool_id
           end
 
-          tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: cloud_desc.arn).tag_list
+          tags = MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).list_tags(arn: cloud_desc.arn).tag_list
           if tags and !tags.empty?
             bok['tags'] = MU.structToHash(tags)
           end
@@ -252,7 +252,7 @@ module MU
               cloud: "AWS",
               credentials: @credentials,
               type: "vpcs",
-              region: @config['region'],
+              region: @region,
               subnets: cloud_desc.vpc_options.subnet_ids.map { |s| { "subnet_id" => s } }
             )
             if cloud_desc.vpc_options.security_group_ids and
@@ -262,7 +262,7 @@ module MU
                   id: sg,
                   cloud: "AWS",
                   credentials: @credentials,
-                  region: @config['region'],
+                  region: @region,
                   type: "firewall_rules",
                 )
               }
@@ -683,7 +683,7 @@ module MU
               params[:log_publishing_options]["SEARCH_SLOW_LOGS"] = {}
               params[:log_publishing_options]["SEARCH_SLOW_LOGS"][:enabled] = true
               params[:log_publishing_options]["SEARCH_SLOW_LOGS"][:cloud_watch_logs_log_group_arn] = arn
-              MU::Cloud.resourceClass("AWS", "Log").allowService("es.amazonaws.com", arn, @config['region'])
+              MU::Cloud.resourceClass("AWS", "Log").allowService("es.amazonaws.com", arn, @region)
             end
           end
 
@@ -813,7 +813,7 @@ module MU
             raise MU::MuError, "Can't tag ElasticSearch domain, cloud descriptor came back without an ARN"
           end
 
-          MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).add_tags(
+          MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).add_tags(
             arn: domain.arn,
             tag_list: tags
           )

data/modules/mu/providers/aws/server.rb

@@ -85,7 +85,7 @@ module MU
             MU::Cloud.fetchUserdata(
               platform: @config["platform"],
               cloud: "AWS",
-              credentials: @config['credentials'],
+              credentials: @credentials,
               template_variables: {
                 "deployKey" => Base64.urlsafe_encode64(@deploy.public_key),
                 "deploySSHKey" => @deploy.ssh_public_key,
@@ -242,8 +242,8 @@ module MU
           else
             MU::Cloud::AWS.createStandardTags(
               instance.instance_id,
-              region: @config['region'],
-              credentials: @config['credentials'],
+              region: @region,
+              credentials: @credentials,
               optional: @config['optional_tags'],
               nametag: @mu_name,
               othertags: @config['tags']
@@ -258,7 +258,7 @@ module MU
               parent_thread_id = Thread.current.object_id
               Thread.new {
                 MU.dupGlobals(parent_thread_id)
-                MU::Cloud::AWS::Server.cleanup(noop: false, ignoremaster: false, region: @config['region'], credentials: @config['credentials'], flags: { "skipsnapshots" => true } )
+                MU::Cloud::AWS::Server.cleanup(noop: false, ignoremaster: false, region: @region, credentials: @credentials, flags: { "skipsnapshots" => true } )
               }
             end
           end
@@ -307,9 +307,9 @@ module MU
           instance_descriptor[:user_data] = Base64.encode64(@userdata)
         end
 
-        MU::Cloud::AWS::Server.waitForAMI(@config["image_id"], region: @config['region'], credentials: @config['credentials'])
+        MU::Cloud::AWS::Server.waitForAMI(@config["image_id"], region: @region, credentials: @credentials)
 
-        instance_descriptor[:block_device_mappings] = MU::Cloud::AWS::Server.configureBlockDevices(image_id: @config["image_id"], storage: @config['storage'], region: @config['region'], credentials: @credentials)
+        instance_descriptor[:block_device_mappings] = MU::Cloud::AWS::Server.configureBlockDevices(image_id: @config["image_id"], storage: @config['storage'], region: @region, credentials: @credentials)
 
         instance_descriptor[:monitoring] = {enabled: @config['monitoring']}
 
@@ -330,10 +330,26 @@ module MU
           resp.nil? or resp.instances.nil? or instance.nil?
         }
 
+        bad_subnets = []
+        mysubnet_ids = if mySubnets
+          mySubnets.map { |s| s.cloud_id }
+        end
         begin
           MU.retrier([Aws::EC2::Errors::InvalidGroupNotFound, Aws::EC2::Errors::InvalidSubnetIDNotFound, Aws::EC2::Errors::InvalidParameterValue], loop_if: loop_if, loop_msg: "Waiting for run_instances to return #{@mu_name}") {
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).run_instances(instance_descriptor)
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).run_instances(instance_descriptor)
           }
+        rescue Aws::EC2::Errors::Unsupported => e
+          bad_subnets << instance_descriptor[:subnet_id]
+          better_subnet = (mysubnet_ids - bad_subnets).sample
+          if e.message !~ /is not supported in your requested Availability Zone/ and
+             (mysubnet_ids.nil? or mysubnet_ids.empty? or
+              mysubnet_ids.size == bad_subnets.size or
+              better_subnet.nil? or better_subnet == "")
+            raise MuError.new e.message, details: mysubnet_ids
+          end
+          instance_descriptor[:subnet_id] = (mysubnet_ids - bad_subnets).sample
+          MU.log "One or more subnets does not support this instance type, attempting with #{instance_descriptor[:subnet_id]} instead", MU::WARN, details: bad_subnets
+          retry
         rescue Aws::EC2::Errors::InvalidRequest => e
           MU.log e.message, MU::ERR, details: instance_descriptor
           raise e
@@ -351,12 +367,12 @@ module MU
         if hard
           groupname = nil
           if !@config['basis'].nil?
-            resp = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_instances(
+            resp = MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_auto_scaling_instances(
               instance_ids: [@cloud_id]
             )
             groupname = resp.auto_scaling_instances.first.auto_scaling_group_name
             MU.log "Pausing Autoscale processes in #{groupname}", MU::NOTICE
-            MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).suspend_processes(
+            MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).suspend_processes(
               auto_scaling_group_name: groupname,
               scaling_processes: [
                 "Terminate",
@@ -365,22 +381,22 @@ module MU
           end
           begin
             MU.log "Stopping #{@mu_name} (#{@cloud_id})", MU::NOTICE
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).stop_instances(
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).stop_instances(
               instance_ids: [@cloud_id]
             )
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).wait_until(:instance_stopped, instance_ids: [@cloud_id]) do |waiter|
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).wait_until(:instance_stopped, instance_ids: [@cloud_id]) do |waiter|
               waiter.before_attempt do
                 MU.log "Waiting for #{@mu_name} to stop for hard reboot"
               end
             end
             MU.log "Starting #{@mu_name} (#{@cloud_id})"
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).start_instances(
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).start_instances(
               instance_ids: [@cloud_id]
             )
           ensure
             if !groupname.nil?
               MU.log "Resuming Autoscale processes in #{groupname}", MU::NOTICE
-              MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).resume_processes(
+              MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).resume_processes(
                 auto_scaling_group_name: groupname,
                 scaling_processes: [
                   "Terminate",
@@ -390,7 +406,7 @@ module MU
           end
         else
           MU.log "Rebooting #{@mu_name} (#{@cloud_id})"
-          MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).reboot_instances(
+          MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).reboot_instances(
             instance_ids: [@cloud_id]
           )
         end
@@ -405,7 +421,7 @@ module MU
         return nil if @config.nil? or @deploy.nil?
 
         nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
-        if !@config["vpc"].nil? and !MU::Cloud.resourceClass("AWS", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
+        if !@config["vpc"].nil? and !MU::Cloud.resourceClass("AWS", "VPC").haveRouteToInstance?(cloud_desc, region: @region, credentials: @credentials)
           if !@nat.nil?
             if @nat.is_a?(Struct) && @nat.nat_gateway_id && @nat.nat_gateway_id.start_with?("nat-")
               raise MuError, "Configured to use NAT Gateway, but I have no route to instance. Either use Bastion, or configure VPC peering"
@@ -449,6 +465,9 @@ module MU
         raise MuError, "Couldn't find instance #{@mu_name} (#{@cloud_id})" if !cloud_desc
         return false if !MU::MommaCat.lock(@cloud_id+"-orchestrate", true)
         return false if !MU::MommaCat.lock(@cloud_id+"-groom", true)
+
+        getIAMProfile
+
         finish = Proc.new { |status|
           MU::MommaCat.unlock(@cloud_id+"-orchestrate")
           MU::MommaCat.unlock(@cloud_id+"-groom")
@@ -457,8 +476,8 @@ module MU
 
         MU::Cloud::AWS.createStandardTags(
           @cloud_id,
-          region: @config['region'],
-          credentials: @config['credentials'],
+          region: @region,
+          credentials: @credentials,
           optional: @config['optional_tags'],
           nametag: @mu_name,
           othertags: @config['tags']
@@ -474,7 +493,15 @@ module MU
         }
         MU.retrier([Aws::EC2::Errors::ServiceError], max: 30, wait: 40, loop_if: loop_if) { |retries, _wait|
           if cloud_desc and cloud_desc.state.name == "terminated"
-            raise MuError, "#{@cloud_id} appears to have been terminated mid-bootstrap!"
+            logs = if !@config['basis'].nil?
+              pool = @deploy.findLitterMate(type: "server_pools", name: @config["name"])
+              if pool
+                MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_scaling_activities(auto_scaling_group_name: pool.cloud_id).activities
+              else
+                nil
+              end
+            end
+            raise MuError.new, "#{@cloud_id} appears to have been terminated mid-bootstrap!", details: logs
           end
           if retries % 3 == 0
             MU.log "Waiting for EC2 instance #{@mu_name} (#{@cloud_id}) to be ready...", MU::NOTICE
@@ -495,7 +522,7 @@ module MU
 
         if !@config['src_dst_check'] and !@config["vpc"].nil?
           MU.log "Disabling source_dest_check #{@mu_name} (making it NAT-worthy)"
-          MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
+          MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).modify_instance_attribute(
             instance_id: @cloud_id,
             source_dest_check: { value: false }
           )
@@ -503,7 +530,7 @@ module MU
 
         # Set console termination protection. Autoscale nodes won't set this
         # by default.
-        MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
+        MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).modify_instance_attribute(
           instance_id: @cloud_id,
           disable_api_termination: { value: true}
         )
@@ -516,7 +543,6 @@ module MU
           notify
         end
 
-        getIAMProfile
         finish.call(false) if !bootstrapGroomer
 
         # Make sure we got our name written everywhere applicable
@@ -574,7 +600,7 @@ module MU
         regions.each { |r|
           searches.each { |search|
             search_threads << Thread.new(search) { |params|
-              MU.retrier([Aws::EC2::Errors::InvalidInstanceIDNotFound], wait: 5, max: 5, ignoreme: [Aws::EC2::Errors::InvalidInstanceIDNotFound]) {
+              MU.retrier([], wait: 5, max: 5, ignoreme: [Aws::EC2::Errors::InvalidInstanceIDNotFound]) {
                 MU::Cloud::AWS.ec2(region: r, credentials: args[:credentials]).describe_instances(params).reservations.each { |resp|
                   next if resp.nil? or resp.instances.nil?
                   resp.instances.each { |i|
@@ -604,9 +630,9 @@ module MU
       def toKitten(**_args)
         bok = {
           "cloud" => "AWS",
-          "credentials" => @config['credentials'],
+          "credentials" => @credentials,
           "cloud_id" => @cloud_id,
-          "region" => @config['region']
+          "region" => @region
         }
 
         if !cloud_desc
@@ -616,7 +642,7 @@ module MU
 
         asgs = MU::Cloud.resourceClass("AWS", "ServerPool").find(
           instance_id: @cloud_id,
-          region: @config['region'],
+          region: @region,
           credentials: @credentials
         )
         if asgs.size > 0
@@ -651,7 +677,7 @@ module MU
 
         bok['image_id'] = cloud_desc.image_id
 
-        ami = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_images(image_ids: [bok['image_id']]).images.first
+        ami = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_images(image_ids: [bok['image_id']]).images.first
 
         if ami.nil? or ami.empty?
           MU.log "#{@mu_name} source image #{bok['image_id']} no longer exists", MU::WARN
@@ -660,7 +686,7 @@ module MU
 
         if cloud_desc.block_device_mappings and !cloud_desc.block_device_mappings.empty?
           vol_map = {}
-          MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_volumes(
+          MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_volumes(
             volume_ids: cloud_desc.block_device_mappings.map { |d| d.ebs.volume_id if d.ebs }
           ).volumes.each { |vol|
             vol_map[vol.volume_id] = vol
@@ -696,7 +722,7 @@ module MU
               id: int.vpc_id,
               cloud: "AWS",
               credentials: @credentials,
-              region: @config['region'],
+              region: @region,
               subnet_id: int.subnet_id,
               habitat: MU::Config::Ref.get(
                 id: int.owner_id,
@@ -725,11 +751,11 @@ module MU
           if int.groups.size > 0
 
             require 'mu/providers/aws/firewall_rule'
-            ifaces = MU::Cloud.resourceClass("AWS", "FirewallRule").getAssociatedInterfaces(int.groups.map { |sg| sg.group_id }, credentials: @credentials, region: @config['region'])
+            ifaces = MU::Cloud.resourceClass("AWS", "FirewallRule").getAssociatedInterfaces(int.groups.map { |sg| sg.group_id }, credentials: @credentials, region: @region)
             done_local_rules = false
             int.groups.each { |sg|
               if !done_local_rules and ifaces[sg.group_id].size == 1
-                sg_desc = MU::Cloud.resourceClass("AWS", "FirewallRule").find(cloud_id: sg.group_id, credentials: @credentials, region: @config['region']).values.first
+                sg_desc = MU::Cloud.resourceClass("AWS", "FirewallRule").find(cloud_id: sg.group_id, credentials: @credentials, region: @region).values.first
                 if sg_desc
                   bok["ingress_rules"] = MU::Cloud.resourceClass("AWS", "FirewallRule").rulesToBoK(sg_desc.ip_permissions)
                   bok["ingress_rules"].concat(MU::Cloud.resourceClass("AWS", "FirewallRule").rulesToBoK(sg_desc.ip_permissions_egress, egress: true))
@@ -743,7 +769,7 @@ module MU
                 cloud: "AWS",
                 credentials: @credentials,
                 type: "firewall_rules",
-                region: @config['region']
+                region: @region
               )
             }
           end
@@ -799,7 +825,7 @@ module MU
           if !@config['chef_data'].nil?
             deploydata.merge!(@config['chef_data'])
           end
-          deploydata["region"] = @config['region'] if !@config['region'].nil?
+          deploydata["region"] = @region if !@region.nil?
           if !@named
             MU::MommaCat.nameKitten(self, no_dns: true)
             @named = true
@@ -883,7 +909,7 @@ module MU
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":instance/"+@cloud_id
+          "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":ec2:"+@region+":"+MU::Cloud::AWS.credToAcct(@credentials)+":instance/"+@cloud_id
         end
 
         @cloud_desc_cache = nil
@@ -896,7 +922,7 @@ module MU
           retries = 0
           if !@cloud_id.nil?
             begin
-              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(instance_ids: [@cloud_id])
+              resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_instances(instance_ids: [@cloud_id])
               if resp and resp.reservations and resp.reservations.first and
                  resp.reservations.first.instances and
                  resp.reservations.first.instances.first
@@ -943,7 +969,7 @@ module MU
           # Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
           # which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
           # The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
-          if MU::Cloud.resourceClass("AWS", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials']) or @deploydata["public_ip_address"].nil?
+          if MU::Cloud.resourceClass("AWS", "VPC").haveRouteToInstance?(cloud_desc, region: @region, credentials: @credentials) or @deploydata["public_ip_address"].nil?
             @config['canonical_ip'] = cloud_desc.private_ip_address
             @deploydata["private_ip_address"] = cloud_desc.private_ip_address
             return cloud_desc.private_ip_address
@@ -1170,7 +1196,7 @@ module MU
           retries = 0
           MU.log "Waiting for Windows instance password to be set by Amazon and flagged as available from the API. Note- if you're using a source AMI that already has its password set, this may fail. You'll want to set use_cloud_provider_windows_password to false if this is the case.", MU::NOTICE
           begin
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).wait_until(:password_data_available, instance_id: @cloud_id) do |waiter|
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).wait_until(:password_data_available, instance_id: @cloud_id) do |waiter|
               waiter.max_attempts = 60
               waiter.before_attempt do |attempts|
                 MU.log "Waiting for Windows password data to be available for node #{@mu_name}", MU::NOTICE if attempts % 5 == 0
@@ -1182,15 +1208,15 @@ module MU
           rescue Aws::Waiters::Errors::TooManyAttemptsError => e
             if retries < 2
               retries = retries + 1
-              MU.log "wait_until(:password_data_available, instance_id: #{@cloud_id}) in #{@config['region']} never got a good response, retrying (#{retries}/2)", MU::WARN, details: e.inspect
+              MU.log "wait_until(:password_data_available, instance_id: #{@cloud_id}) in #{@region} never got a good response, retrying (#{retries}/2)", MU::WARN, details: e.inspect
               retry
             else
-              MU.log "wait_until(:password_data_available, instance_id: #{@cloud_id}) in #{@config['region']} never returned- this image may not be configured to have its password set by AWS.", MU::ERR
+              MU.log "wait_until(:password_data_available, instance_id: #{@cloud_id}) in #{@region} never returned- this image may not be configured to have its password set by AWS.", MU::ERR
               return nil
             end
           end
 
-          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).get_password_data(instance_id: @cloud_id)
+          resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).get_password_data(instance_id: @cloud_id)
           encrypted_password = resp.password_data
 
           # Note: This is already implemented in the decrypt_windows_password API call
@@ -1209,7 +1235,7 @@ module MU
         # instead of VPC.
         # @param ip [String]: Request a specific IP address.
         # @param region [String]: The cloud provider region
-        def self.findFreeElasticIp(classic: false, ip: nil, region: MU.curRegion)
+        def self.findFreeElasticIp(classic: false, ip: nil, region: MU.curRegion, credentials: nil)
           filters = Array.new
           if !classic
             filters << {name: "domain", values: ["vpc"]}
@@ -1219,25 +1245,22 @@ module MU
           filters << {name: "public-ip", values: [ip]} if ip != nil
 
           if filters.size > 0
-            resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(filters: filters)
+            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_addresses(filters: filters)
           else
-            resp = MU::Cloud::AWS.ec2(region: region).describe_addresses()
+            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_addresses
           end
           resp.addresses.each { |address|
-            return address if (address.network_interface_id.nil? || address.network_interface_id.empty?) && !@eips_used.include?(address.public_ip)
+            return address if (address.network_interface_id.nil? or address.network_interface_id.empty?) or !@eips_used.include?(address.public_ip)
           }
-          if ip != nil
-            if !classic
-              raise MuError, "Requested EIP #{ip}, but no such IP exists or is avaulable in VPC"
-            else
-              raise MuError, "Requested EIP #{ip}, but no such IP exists or is available in EC2 Classic"
-            end
+          if !ip.nil?
+            mode = classic ? "EC2 Classic" : "VPC"
+            raise MuError.new "Requested EIP #{ip}, but no such IP exists or is available in #{mode} mode#{credentials ? " with credentials #{credentials}" : ""}", details: { "describe_address filters" => filters, "describe_address response" => resp }
           end
           if !classic
-            resp = MU::Cloud::AWS.ec2(region: region).allocate_address(domain: "vpc")
+            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).allocate_address(domain: "vpc")
             new_ip = resp.public_ip
           else
-            new_ip = MU::Cloud::AWS.ec2(region: region).allocate_address().public_ip
+            new_ip = MU::Cloud::AWS.ec2(region: region, credentials: credentials).allocate_address().public_ip
           end
           filters = [{name: "public-ip", values: [new_ip]}]
           if resp.domain
@@ -1253,8 +1276,8 @@ module MU
           begin
             begin
               sleep 5
-              resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(
-                  filters: filters
+              resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_addresses(
+                filters: filters
               )
               addr = resp.addresses.first
             end while resp.addresses.size < 1 or addr.public_ip.nil?
@@ -1275,19 +1298,19 @@ module MU
         def addVolume(dev, size, type: "gp2", delete_on_termination: false)
 
           if setDeleteOntermination(dev, delete_on_termination)
-            MU.log "A volume #{device} already attached to #{self}, skipping", MU::NOTICE
+            MU.log "A volume #{dev} already attached to #{self}, skipping", MU::NOTICE
             return
           end
 
           MU.log "Creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
-          creation = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_volume(
+          creation = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_volume(
             availability_zone: cloud_desc.placement.availability_zone,
             size: size,
             volume_type: type
           )
 
           MU.retrier(wait: 3, loop_if: Proc.new {
-            creation = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(volume_ids: [creation.volume_id]).volumes.first
+            creation = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_volumes(volume_ids: [creation.volume_id]).volumes.first
             if !["creating", "available"].include?(creation.state)
               raise MuError, "Saw state '#{creation.state}' while creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
             end
@@ -1298,27 +1321,35 @@ module MU
           if @deploy
             MU::Cloud::AWS.createStandardTags(
               creation.volume_id,
-              region: @config['region'],
-              credentials: @config['credentials'],
+              region: @region,
+              credentials: @credentials,
               optional: @config['optional_tags'],
               nametag: @mu_name+"-"+dev.upcase,
               othertags: @config['tags']
             )
           end
 
-          attachment = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_volume(
-            device: dev,
-            instance_id: @cloud_id,
-            volume_id: creation.volume_id
-          )
+          MU.log "Attaching #{creation.volume_id} as #{dev} to #{@cloud_id} in #{@region} (credentials #{@credentials})"
+          attachment = nil
+          MU.retrier([Aws::EC2::Errors::IncorrectState], wait: 15, max: 4) {
+            attachment = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).attach_volume(
+              device: dev,
+              instance_id: @cloud_id,
+              volume_id: creation.volume_id
+            )
+          }
 
           begin
-            sleep 3
-            attachment = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(volume_ids: [attachment.volume_id]).volumes.first.attachments.first
-            if !["attaching", "attached"].include?(attachment.state)
-              raise MuError, "Saw state '#{creation.state}' while creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
+            att_resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_volumes(volume_ids: [attachment.volume_id])
+            if att_resp and att_resp.volumes and !att_resp.volumes.empty? and
+               att_resp.volumes.first.attachments and
+               !att_resp.volumes.first.attachments.empty?
+              attachment = att_resp.volumes.first.attachments.first
+              if !attachment.nil? and !["attaching", "attached"].include?(attachment.state)
+                raise MuError, "Saw state '#{creation.state}' while creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
+              end
             end
-          end while attachment.state != "attached"
+          end while attachment.nil? or attachment.state != "attached"
 
           # Set delete_on_termination, which for some reason is an instance
           # attribute and not on the attachment
@@ -1334,7 +1365,7 @@ module MU
             return true
           end
           begin
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_instances(
                 instance_ids: [@cloud_id]
             ).reservations.each { |resp|
               if !resp.nil? and !resp.instances.nil?
@@ -1385,7 +1416,7 @@ module MU
                 }
               end
             end
-            elastic_ip = findFreeElasticIp(classic: classic, ip: ip)
+            elastic_ip = findFreeElasticIp(classic: classic, ip: ip, credentials: credentials)
             if !ip.nil? and (elastic_ip.nil? or ip != elastic_ip.public_ip)
               raise MuError, "Requested EIP #{ip}, but this IP does not exist or is not available"
             end
@@ -1737,6 +1768,7 @@ module MU
             return size
           end
 
+
           return size if types.has_key?(size)
 
           if size.nil? or !types.has_key?(size)
@@ -1782,7 +1814,8 @@ module MU
         def self.generateStandardRole(server, configurator)
           role = {
             "name" => server["name"],
-            "credentials" => server["credentials"],
+            "bare_policies" => !server['generate_iam_role'],
+            "strip_path" => server["role_strip_path"],
             "can_assume" => [
               {
                 "entity_id" => "ec2.amazonaws.com",
@@ -1801,6 +1834,7 @@ module MU
               }
             ]
           }
+          role["credentials"] = server["credentials"] if server["credentials"]
           if server['iam_policies']
             role['iam_policies'] = server['iam_policies'].dup
           end
@@ -1834,9 +1868,10 @@ module MU
               MU.log "Cannot mix iam_policies with generate_iam_role set to false", MU::ERR
               ok = false
             end
-          else
-            generateStandardRole(server, configurator)
           end
+
+          generateStandardRole(server, configurator)
+
           if !server['create_image'].nil?
             if server['create_image'].has_key?('copy_to_regions') and
                 (server['create_image']['copy_to_regions'].nil? or
@@ -2085,7 +2120,7 @@ module MU
         def haveElasticIP?
           if !cloud_desc.public_ip_address.nil?
             begin
-              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(public_ips: [cloud_desc.public_ip_address])
+              resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_addresses(public_ips: [cloud_desc.public_ip_address])
               if resp.addresses.size > 0 and resp.addresses.first.instance_id == @cloud_id
                 return true
               end
@@ -2100,9 +2135,9 @@ module MU
         def configureNetworking
           if !@config['static_ip'].nil?
             if !@config['static_ip']['ip'].nil?
-              MU::Cloud::AWS::Server.associateElasticIp(@cloud_id, classic: @vpc.nil?, ip: @config['static_ip']['ip'])
+              MU::Cloud::AWS::Server.associateElasticIp(@cloud_id, classic: @vpc.nil?, ip: @config['static_ip']['ip'], credentials: @credentials)
             elsif !haveElasticIP?
-              MU::Cloud::AWS::Server.associateElasticIp(@cloud_id, classic: @vpc.nil?)
+              MU::Cloud::AWS::Server.associateElasticIp(@cloud_id, classic: @vpc.nil?, credentials: @credentials)
             end
           end
 
@@ -2110,7 +2145,7 @@ module MU
             subnet = @vpc.getSubnet(cloud_id: cloud_desc.subnet_id)
 
             _nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
-            if subnet.private? and !nat_ssh_host and !MU::Cloud.resourceClass("AWS", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
+            if subnet.private? and !nat_ssh_host and !MU::Cloud.resourceClass("AWS", "VPC").haveRouteToInstance?(cloud_desc, region: @region, credentials: @credentials)
               raise MuError, "#{@mu_name} is in a private subnet (#{subnet}), but has no bastion host configured, and I have no other route to it"
             end
 
@@ -2127,17 +2162,17 @@ module MU
                   next
                 end
                 MU.log "Adding network interface on subnet #{s.cloud_id} for #{@mu_name}"
-                iface = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_network_interface(subnet_id: s.cloud_id).network_interface
+                iface = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_network_interface(subnet_id: s.cloud_id).network_interface
                 MU::Cloud::AWS.createStandardTags(
                   iface.network_interface_id,
-                  region: @config['region'],
-                  credentials: @config['credentials'],
+                  region: @region,
+                  credentials: @credentials,
                   optional: @config['optional_tags'],
                   nametag: @mu_name+"-ETH"+device_index.to_s,
                   othertags: @config['tags']
                 )
 
-                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_network_interface(
+                MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).attach_network_interface(
                   network_interface_id: iface.network_interface_id,
                   instance_id: cloud_desc.instance_id,
                   device_index: device_index
@@ -2156,7 +2191,7 @@ module MU
             cloud_desc.network_interfaces.each { |int|
               if int.private_ip_address == cloud_desc.private_ip_address and int.private_ip_addresses.size < (@config['add_private_ips'] + 1)
                 MU.log "Adding #{@config['add_private_ips']} extra private IP addresses to #{cloud_desc.instance_id}"
-                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).assign_private_ip_addresses(
+                MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).assign_private_ip_addresses(
                   network_interface_id: int.network_interface_id,
                   secondary_private_ip_address_count: @config['add_private_ips'],
                   allow_reassignment: false
@@ -2167,14 +2202,14 @@ module MU
         end
 
         def tagVolumes
-          volumes = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(filters: [name: "attachment.instance-id", values: [@cloud_id]])
+          volumes = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_volumes(filters: [name: "attachment.instance-id", values: [@cloud_id]])
           volumes.each { |vol|
             vol.volumes.each { |volume|
               volume.attachments.each { |attachment|
                 MU::Cloud::AWS.createStandardTags(
                   attachment.volume_id,
-                  region: @config['region'],
-                  credentials: @config['credentials'],
+                  region: @region,
+                  credentials: @credentials,
                   optional: @config['optional_tags'],
                   nametag: ["/dev/sda", "/dev/sda1"].include?(attachment.device) ? "ROOT-"+@mu_name : @mu_name+"-"+attachment.device.upcase,
                   othertags: @config['tags']
@@ -2195,7 +2230,7 @@ module MU
               alarm_obj = MU::MommaCat.findStray(
                 "AWS",
                 "alarms",
-                region: @config["region"],
+                region: @region,
                 deploy_id: @deploy.deploy_id,
                 name: alarm['name']
               ).first
@@ -2204,8 +2239,8 @@ module MU
               if alarm["enable_notifications"]
                 # XXX vile, this should be a sibling resource generated by the
                 # parser
-                topic_arn = MU::Cloud.resourceClass("AWS", "Notification").createTopic(alarm["notification_group"], region: @config["region"], credentials: @config['credentials'])
-                MU::Cloud.resourceClass("AWS", "Notification").subscribe(topic_arn, alarm["notification_endpoint"], alarm["notification_type"], region: @config["region"], credentials: @config["credentials"])
+                topic_arn = MU::Cloud.resourceClass("AWS", "Notification").createTopic(alarm["notification_group"], region: @region, credentials: @credentials)
+                MU::Cloud.resourceClass("AWS", "Notification").subscribe(topic_arn, alarm["notification_endpoint"], alarm["notification_type"], region: @region, credentials: @credentials)
                 alarm["alarm_actions"] = [topic_arn]
                 alarm["ok_actions"]  = [topic_arn]
               end
@@ -2226,36 +2261,88 @@ module MU
                 evaluation_periods: alarm["evaluation_periods"],
                 threshold: alarm["threshold"],
                 comparison_operator: alarm["comparison_operator"],
-                region: @config["region"],
-                credentials: @config['credentials']
+                region: @region,
+                credentials: @credentials
               )
             }
           end
         end
 
-        # We have issues sometimes where our dns_records are pointing at the wrong node name and IP address.
-
         def getIAMProfile
-          arn = if @config['generate_iam_role']
-            role = @deploy.findLitterMate(name: @config['name'], type: "roles")
-            s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
-              'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(@credentials)+'/'+file
+          self.class.getIAMProfile(
+            @config['name'],
+            @deploy,
+            generated: @config['generate_iam_role'],
+            role_name: @config['iam_role'],
+            region: @region,
+            credentials: @credentials,
+            want_arn: true
+          )
+        end
+
+# XXX move to public section
+        def self.getIAMProfile(myname, deploy, generated: true, role_name: nil, region: nil, credentials: nil, want_arn: false)
+
+          arn = if generated
+            role = deploy.findLitterMate(name: myname, type: "roles", debug: true)
+            if !role
+              raise MuError, "Failed to find a role matching #{myname}"
+            end
+            s3_objs = ["#{deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
+              'arn:'+(MU::Cloud::AWS.isGovCloud?(region) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/'+file
             }
-            MU.log "Adding S3 read permissions to #{@mu_name}'s IAM profile", MU::NOTICE, details: s3_objs
+            MU.log "Adding S3 read permissions to #{myname}'s IAM profile", MU::NOTICE, details: s3_objs
             role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
   
-            @config['iam_role'] = role.mu_name
+            role_name = role.mu_name
             role.cloudobj.createInstanceProfile
   
-          elsif @config['iam_role'].nil?
-            raise MuError, "#{@mu_name} has generate_iam_role set to false, but no iam_role assigned."
+          elsif role_name.nil?
+            raise MuError, "#{myname} has generate_iam_role set to false, but no iam_role assigned."
+          else
+            begin
+              ext_prof = MU::Cloud::AWS.iam(credentials: credentials).get_instance_profile(instance_profile_name: role_name)
+              role_name = ext_prof.instance_profile.instance_profile_name
+              ext_prof.instance_profile.arn
+            rescue Aws::IAM::Errors::NoSuchEntity
+              role = MU::MommaCat.findStray("AWS", "role", cloud_id: role_name, dummy_ok: true, credentials: credentials).first
+              if !role
+                raise MuError, "#{myname} specified iam_role '#{role_name}', but I can't find a role with that name to use when creating an instance profile"
+              end
+              role.cloudobj.createInstanceProfile
+            end
           end
 
-          if !@config["iam_role"].nil?
-            if arn
+          role_or_policy = deploy.findLitterMate(name: myname, type: "roles")
+
+          # Make sure our permissions to read our identity secrets are set
+          s3_objs = [
+            "#{deploy.deploy_id}-secret",
+            "#{role_or_policy.mu_name}.pfx",
+            "#{role_or_policy.mu_name}.crt",
+            "#{role_or_policy.mu_name}.key",
+            "#{role_or_policy.mu_name}-winrm.crt",
+            "#{role_or_policy.mu_name}-winrm.key"].map { |file| 
+              'arn:'+(MU::Cloud::AWS.isGovCloud?(region) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/'+file
+            }
+          if generated
+            role_or_policy.injectPolicyTargets("MuSecrets", s3_objs)
+          elsif role_name
+            realrole = MU::MommaCat.findStray("AWS", "role", cloud_id: role_name, dummy_ok: true, credentials: credentials).first
+            if !role_or_policy
+              raise MuError, "I should have a bare policy littermate named #{name} but I can't find it"
+            end
+            if realrole
+              role_or_policy.bindTo("role", realrole.cloud_id)
+              realrole.injectPolicyTargets(role_or_policy.mu_name+"-MUSECRETS", s3_objs)
+            end
+          end
+
+          if !role_name.nil?
+            if arn and want_arn
               return {arn: arn}
             else
-              return {name: @config["iam_role"]}
+              return {name: role_name}
             end
           end
 
@@ -2272,8 +2359,8 @@ module MU
             if vol[:device_name] == device
               if vol[:ebs][:delete_on_termination] != delete_on_termination
                 vol[:ebs][:delete_on_termination] = delete_on_termination
-                MU.log "Setting delete_on_termination flag to #{delete_on_termination.to_s} on #{@mu_name}'s #{dev}"
-                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
+                MU.log "Setting delete_on_termination flag to #{delete_on_termination.to_s} on #{@mu_name}'s #{device}"
+                MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).modify_instance_attribute(
                   instance_id: @cloud_id,
                   block_device_mappings: mappings
                 )
@@ -2317,16 +2404,17 @@ module MU
               exclude_storage: img_cfg['image_exclude_storage'],
               copy_to_regions: img_cfg['copy_to_regions'],
               make_public: img_cfg['public'],
-              region: @config['region'],
+              region: @region,
               tags: @config['tags'],
-              credentials: @config['credentials']
+              credentials: @credentials
           )
+
           @deploy.notify("images", @config['name'], ami_ids)
           @config['image_created'] = true
           if img_cfg['image_then_destroy']
-            MU::Cloud::AWS::Server.waitForAMI(ami_ids[@config['region']], region: @config['region'], credentials: @config['credentials'])
-            MU.log "AMI #{ami_ids[@config['region']]} ready, removing source node #{@mu_name}"
-            MU::Cloud::AWS::Server.terminateInstance(id: @cloud_id, region: @config['region'], deploy_id: @deploy.deploy_id, mu_name: @mu_name, credentials: @config['credentials'])
+            MU::Cloud::AWS::Server.waitForAMI(ami_ids[@region], region: @region, credentials: @credentials)
+            MU.log "AMI #{ami_ids[@region]} ready, removing source node #{@mu_name}"
+            MU::Cloud::AWS::Server.terminateInstance(id: @cloud_id, region: @region, deploy_id: @deploy.deploy_id, mu_name: @mu_name, credentials: @credentials)
             destroy
           end
         end