cloud-mu 3.1.4 → 3.1.5

data/modules/mu/clouds/aws/role.rb

@@ -201,7 +201,11 @@ module MU
         def arn
           desc = cloud_desc
           if desc["role"]
-            desc["role"].arn
+            if desc['role'].is_a?(Hash)
+              desc["role"][:arn] # why though
+            else
+              desc["role"].arn
+            end
           else
             nil
           end
@@ -290,21 +294,21 @@ end
           if !policy.match(/^#{@deploy.deploy_id}/)
             policy = @mu_name+"-"+policy.upcase
           end
-
-          my_policies = cloud_desc["policies"]
+          my_policies = cloud_desc(use_cache: false)["policies"]
           my_policies ||= []
-          
+
+          seen_policy = false
           my_policies.each { |p|
             if p.policy_name == policy
+              seen_policy = true
               old = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy_version(
                 policy_arn: p.arn,
                 version_id: p.default_version_id
               ).policy_version
 
               doc = JSON.parse URI.decode_www_form_component old.document
-              
               need_update = false
-              
+
               doc["Statement"].each { |s|
                 targets.each { |target|
                   target_string = target
@@ -333,6 +337,10 @@ end
               end
             end
           }
+
+          if !seen_policy
+            MU.log "Was given new targets for policy #{policy}, but I don't see any such policy attached to role #{@cloud_id}", MU::WARN, details: targets
+          end
         end
 
         # Delete an IAM policy, along with attendant versions and attachments.
@@ -525,7 +533,7 @@ end
               end
             rescue ::Aws::IAM::Errors::NoSuchEntity
             end
-            
+
           else
             marker = nil
             begin
@@ -614,7 +622,7 @@ end
 
             return bok if @config['bare_policies']
           end
-          
+
           if desc.tags and desc.tags.size > 0
             bok["tags"] = MU.structToHash(desc.tags, stringify_keys: true)
           end
@@ -838,6 +846,7 @@ end
           else
             raise MuError, "Invalid entitytype '#{entitytype}' passed to MU::Cloud::AWS::Role.bindTo. Must be be one of: user, group, role, instance_profile"
           end
+          cloud_desc(use_cache: false)
         end
 
         # Create an instance profile for EC2 instances, named identically and

data/modules/mu/clouds/aws/search_domain.rb

@@ -693,7 +693,7 @@ module MU
           interval = 60
 
           begin
-            resp = cloud_desc
+            resp = cloud_desc(use_cache: false)
 
             if (resp.endpoint.nil? or resp.endpoint.empty?) and
                (resp.endpoints.nil? or resp.endpoints.empty?) and

data/modules/mu/clouds/aws/server.rb

@@ -240,8 +240,14 @@ module MU
             end
             MU::MommaCat.unlock(instance.instance_id+"-create")
           else
-            MU::Cloud::AWS.createStandardTags(instance.instance_id, region: @config['region'], credentials: @config['credentials'])
-            MU::Cloud::AWS.createTag(instance.instance_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
+            MU::Cloud::AWS.createStandardTags(
+              instance.instance_id,
+              region: @config['region'],
+              credentials: @config['credentials'],
+              optional: @config['optional_tags'],
+              nametag: @mu_name,
+              othertags: @config['tags']
+            )
           end
           done = true
         rescue StandardError => e
@@ -262,14 +268,11 @@ module MU
         return @config
       end
 
-
-
       # Create an Amazon EC2 instance.
       def createEc2Instance
-        node = @config['mu_name']
 
         instance_descriptor = {
-          :image_id => @config["ami_id"],
+          :image_id => @config["image_id"],
           :key_name => @deploy.ssh_key_name,
           :instance_type => @config["size"],
           :disable_api_termination => true,
@@ -277,62 +280,25 @@ module MU
           :max_count => 1
         }
 
-        arn = nil
-        if @config['generate_iam_role']
-          role = @deploy.findLitterMate(name: @config['name'], type: "roles")
-          s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
-            'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(@credentials)+'/'+file
-          }
-          MU.log "Adding S3 read permissions to #{@mu_name}'s IAM profile", MU::NOTICE, details: s3_objs
-          role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
-
-          @config['iam_role'] = role.mu_name
-          arn = role.cloudobj.createInstanceProfile
-#            @cfm_role_name, @cfm_prof_name
-
-        elsif @config['iam_role'].nil?
-          raise MuError, "#{@mu_name} has generate_iam_role set to false, but no iam_role assigned."
-        end
-        if !@config["iam_role"].nil?
-          if arn
-            instance_descriptor[:iam_instance_profile] = {arn: arn}
-          else
-            instance_descriptor[:iam_instance_profile] = {name: @config["iam_role"]}
-          end
-        end
-
-        security_groups = []
-        if @dependencies.has_key?("firewall_rule")
-          @dependencies['firewall_rule'].values.each { |sg|
-            security_groups << sg.cloud_id
-          }
-        end
+        instance_descriptor[:iam_instance_profile] = getIAMProfile
 
+        security_groups = myFirewallRules.map { |fw| fw.cloud_id }
         if security_groups.size > 0
           instance_descriptor[:security_group_ids] = security_groups
         else
           raise MuError, "Didn't get any security groups assigned to be in #{@mu_name}, that shouldn't happen"
         end
 
-        if !@config['private_ip'].nil?
+        if @config['private_ip']
           instance_descriptor[:private_ip_address] = @config['private_ip']
         end
 
         if !@vpc.nil? and @config.has_key?("vpc")
-          subnet_conf = @config['vpc']
-          subnet_conf = @config['vpc']['subnets'].first if @config['vpc'].has_key?("subnets") and !@config['vpc']['subnets'].empty?
-          tag_key, tag_value = subnet_conf['tag'].split(/=/, 2) if !subnet_conf['tag'].nil?
-
-          subnet = @vpc.getSubnet(
-            cloud_id: subnet_conf['subnet_id'],
-            name: subnet_conf['subnet_name'],
-            tag_key: tag_key,
-            tag_value: tag_value
-          )
+          subnet = mySubnets.sample
           if subnet.nil?
-            raise MuError, "Got null subnet id out of #{subnet_conf['vpc']}"
+            raise MuError, "Got null subnet id out of #{@config['vpc']}"
           end
-          MU.log "Deploying #{node} into VPC #{@vpc.cloud_id} Subnet #{subnet.cloud_id}"
+          MU.log "Deploying #{@mu_name} into VPC #{@vpc.cloud_id} Subnet #{subnet.cloud_id}"
           punchAdminNAT
           instance_descriptor[:subnet_id] = subnet.cloud_id
         end
@@ -341,37 +307,10 @@ module MU
           instance_descriptor[:user_data] = Base64.encode64(@userdata)
         end
 
-        MU::Cloud::AWS::Server.waitForAMI(@config["ami_id"], region: @config['region'], credentials: @config['credentials'])
+        MU::Cloud::AWS::Server.waitForAMI(@config["image_id"], region: @config['region'], credentials: @config['credentials'])
 
-        # Figure out which devices are embedded in the AMI already.
-        image = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_images(image_ids: [@config["ami_id"]]).images.first
-        ext_disks = {}
-        if !image.block_device_mappings.nil?
-          image.block_device_mappings.each { |disk|
-            if !disk.device_name.nil? and !disk.device_name.empty? and !disk.ebs.nil? and !disk.ebs.empty?
-              ext_disks[disk.device_name] = MU.structToHash(disk.ebs)
-            end
-          }
-        end
-
-        configured_storage = Array.new
-        if @config["storage"]
-          @config["storage"].each { |vol|
-            # Drop the "encrypted" flag if a snapshot for this device exists
-            # in the AMI, even if they both agree about the value of said
-            # flag. Apparently that's a thing now.
-            if ext_disks.has_key?(vol["device"])
-              if ext_disks[vol["device"]].has_key?(:snapshot_id)
-                vol.delete("encrypted")
-              end
-            end
-            mapping, _cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
-            configured_storage << mapping
-          }
-        end
+        instance_descriptor[:block_device_mappings] = MU::Cloud::AWS::Server.configureBlockDevices(image_id: @config["image_id"], storage: @config['storage'], region: @config['region'], credentials: @credentials)
 
-        instance_descriptor[:block_device_mappings] = configured_storage
-        instance_descriptor[:block_device_mappings].concat(@ephemeral_mappings)
         instance_descriptor[:monitoring] = {enabled: @config['monitoring']}
 
         if @tags and @tags.size > 0
@@ -383,37 +322,24 @@ module MU
           }]
         end
 
-        MU.log "Creating EC2 instance #{node}"
-        MU.log "Instance details for #{node}: #{instance_descriptor}", MU::DEBUG
-#				if instance_descriptor[:block_device_mappings].empty?
-#					instance_descriptor.delete(:block_device_mappings)
-#				end
+        MU.log "Creating EC2 instance #{@mu_name}", details: instance_descriptor
 
-        retries = 0
-        instance = begin
-          response = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).run_instances(instance_descriptor)
-          if response and response.instances and response.instances.size > 0
-            response.instances.first
-          else
-            MU.log "Got a confusing response from run_instances", MU::ERR, details: response
-          end
+        instance = resp = nil
+        loop_if = Proc.new {
+          instance = resp.instances.first if resp and resp.instances
+          resp.nil? or resp.instances.nil? or instance.nil?
+        }
+
+        begin
+          MU.retrier([Aws::EC2::Errors::InvalidGroupNotFound, Aws::EC2::Errors::InvalidSubnetIDNotFound, Aws::EC2::Errors::InvalidParameterValue], loop_if: loop_if, loop_msg: "Waiting for run_instances to return #{@mu_name}") {
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).run_instances(instance_descriptor)
+          }
         rescue Aws::EC2::Errors::InvalidRequest => e
           MU.log e.message, MU::ERR, details: instance_descriptor
           raise e
-        rescue Aws::EC2::Errors::InvalidGroupNotFound, Aws::EC2::Errors::InvalidSubnetIDNotFound, Aws::EC2::Errors::InvalidParameterValue => e
-          if retries < 10
-            if retries > 7
-              MU.log "Seeing #{e.inspect} while trying to launch #{node}, retrying a few more times...", MU::WARN, details: instance_descriptor
-            end
-            sleep 10
-            retries = retries + 1
-            retry
-          else
-            raise MuError, e.inspect
-          end
         end
 
-        MU.log "#{node} (#{instance.instance_id}) coming online"
+        MU.log "#{@mu_name} (#{instance.instance_id}) coming online"
 
         instance
       end
@@ -517,445 +443,82 @@ module MU
       # Apply tags, bootstrap our configuration management, and other
       # administravia for a new instance.
       def postBoot(instance_id = nil)
-        if !instance_id.nil?
-          @cloud_id = instance_id
-        end
+        @cloud_id ||= instance_id
         node, _config, deploydata = describe(cloud_id: @cloud_id)
-        instance = cloud_desc
-        raise MuError, "Couldn't find instance #{@mu_name} (#{@cloud_id})" if !instance
-        @cloud_id = instance.instance_id
-        return false if !MU::MommaCat.lock(instance.instance_id+"-orchestrate", true)
-        return false if !MU::MommaCat.lock(instance.instance_id+"-groom", true)
-
-        MU::Cloud::AWS.createStandardTags(instance.instance_id, region: @config['region'], credentials: @config['credentials'])
-        MU::Cloud::AWS.createTag(instance.instance_id, "Name", node, region: @config['region'], credentials: @config['credentials'])
-
-        if @config['optional_tags']
-          MU::MommaCat.listOptionalTags.each { |key, value|
-            MU::Cloud::AWS.createTag(instance.instance_id, key, value, region: @config['region'], credentials: @config['credentials'])
-          }
-        end
+        @mu_name ||= node
 
-        if !@config['tags'].nil?
-          @config['tags'].each { |tag|
-            MU::Cloud::AWS.createTag(instance.instance_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
-          }
-        end
-        MU.log "Tagged #{node} (#{instance.instance_id}) with MU-ID=#{MU.deploy_id}", MU::DEBUG
+        raise MuError, "Couldn't find instance #{@mu_name} (#{@cloud_id})" if !cloud_desc
+        return false if !MU::MommaCat.lock(@cloud_id+"-orchestrate", true)
+        return false if !MU::MommaCat.lock(@cloud_id+"-groom", true)
+        finish = Proc.new { |status|
+          MU::MommaCat.unlock(@cloud_id+"-orchestrate")
+          MU::MommaCat.unlock(@cloud_id+"-groom")
+          return status
+        }
+
+        MU::Cloud::AWS.createStandardTags(
+          @cloud_id,
+          region: @config['region'],
+          credentials: @config['credentials'],
+          optional: @config['optional_tags'],
+          nametag: @mu_name,
+          othertags: @config['tags']
+        )
 
         # Make double sure we don't lose a cached mu_windows_name value.
-        if windows? or !@config['active_directory'].nil?
-          if @mu_windows_name.nil?
-            @mu_windows_name = deploydata['mu_windows_name']
-          end
+        if (windows? or !@config['active_directory'].nil?)
+          @mu_windows_name ||= deploydata['mu_windows_name']
         end
 
-        retries = -1
-        max_retries = 30
-        begin
-          if instance.nil? or instance.state.name != "running"
-            retries = retries + 1
-            if !instance.nil? and instance.state.name == "terminated"
-              raise MuError, "#{@cloud_id} appears to have been terminated mid-bootstrap!"
-            end
-            if retries % 3 == 0
-              MU.log "Waiting for EC2 instance #{node} (#{@cloud_id}) to be ready...", MU::NOTICE
-            end
-            sleep 40
-            # Get a fresh AWS descriptor
-            instance = MU::Cloud::Server.find(cloud_id: @cloud_id, region: @config['region'], credentials: @config['credentials']).values.first
-            if instance and instance.state.name == "terminated"
-              raise MuError, "EC2 instance #{node} (#{@cloud_id}) terminating during bootstrap!"
-            end
+        loop_if = Proc.new {
+          !cloud_desc(use_cache: false) or cloud_desc.state.name != "running"
+        }
+        MU.retrier([Aws::EC2::Errors::ServiceError], max: 30, wait: 40, loop_if: loop_if) { |retries, _wait|
+          if cloud_desc and cloud_desc.state.name == "terminated"
+            raise MuError, "#{@cloud_id} appears to have been terminated mid-bootstrap!"
           end
-        rescue Aws::EC2::Errors::ServiceError => e
-          if retries < max_retries
-            MU.log "Got #{e.inspect} during initial instance creation of #{@cloud_id}, retrying...", MU::NOTICE, details: instance
-            retries = retries + 1
-            retry
-          else
-            raise MuError, "Too many retries creating #{node} (#{e.inspect})"
+          if retries % 3 == 0
+            MU.log "Waiting for EC2 instance #{@mu_name} (#{@cloud_id}) to be ready...", MU::NOTICE
           end
-        end while instance.nil? or (instance.state.name != "running" and retries < max_retries)
+        }
 
         punchAdminNAT
 
-
-        # If we came up via AutoScale, the Alarm module won't have had our
-        # instance ID to associate us with itself. So invoke that here.
-        # XXX might be possible to do this with regular alarm resources and
-        # dependencies now
-        if !@config['basis'].nil? and @config["alarms"] and !@config["alarms"].empty?
-          @config["alarms"].each { |alarm|
-            alarm_obj = MU::MommaCat.findStray(
-              "AWS",
-              "alarms",
-              region: @config["region"],
-              deploy_id: @deploy.deploy_id,
-              name: alarm['name']
-            ).first
-            alarm["dimensions"] = [{:name => "InstanceId", :value => @cloud_id}]
-
-            if alarm["enable_notifications"]
-              topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"], credentials: @config['credentials'])
-              MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"], credentials: @config["credentials"])
-              alarm["alarm_actions"] = [topic_arn]
-              alarm["ok_actions"]  = [topic_arn]
-            end
-
-            alarm_name = alarm_obj ? alarm_obj.cloud_id : "#{node}-#{alarm['name']}".upcase
-
-            MU::Cloud::AWS::Alarm.setAlarm(
-              name: alarm_name,
-              ok_actions: alarm["ok_actions"],
-              alarm_actions: alarm["alarm_actions"],
-              insufficient_data_actions: alarm["no_data_actions"],
-              metric_name: alarm["metric_name"],
-              namespace: alarm["namespace"],
-              statistic: alarm["statistic"],
-              dimensions: alarm["dimensions"],
-              period: alarm["period"],
-              unit: alarm["unit"],
-              evaluation_periods: alarm["evaluation_periods"],
-              threshold: alarm["threshold"],
-              comparison_operator: alarm["comparison_operator"],
-              region: @config["region"],
-              credentials: @config['credentials']
-            )
-          }
-        end
-
-        # We have issues sometimes where our dns_records are pointing at the wrong node name and IP address.
-        # Make sure that doesn't happen. Happens with server pools only
-        if @config['dns_records'] && !@config['dns_records'].empty?
-          @config['dns_records'].each { |dnsrec|
-            if dnsrec.has_key?("name")
-              if dnsrec['name'].start_with?(MU.deploy_id.downcase) && !dnsrec['name'].start_with?(node.downcase)
-                MU.log "DNS records for #{node} seem to be wrong, deleting from current config", MU::WARN, details: dnsrec
-                dnsrec.delete('name')
-                dnsrec.delete('target')
-              end
-            end
-          }
-        end
+        setAlarms
 
         # Unless we're planning on associating a different IP later, set up a
         # DNS entry for this thing and let it sync in the background. We'll come
         # back to it later.
-        if @config['static_ip'].nil? && !@named
+        if @config['static_ip'].nil? and !@named
           MU::MommaCat.nameKitten(self)
           @named = true
         end
 
         if !@config['src_dst_check'] and !@config["vpc"].nil?
-          MU.log "Disabling source_dest_check #{node} (making it NAT-worthy)"
+          MU.log "Disabling source_dest_check #{@mu_name} (making it NAT-worthy)"
           MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
-              instance_id: @cloud_id,
-              source_dest_check: {:value => false}
+            instance_id: @cloud_id,
+            source_dest_check: { value: false }
           )
         end
 
         # Set console termination protection. Autoscale nodes won't set this
         # by default.
         MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
-            instance_id: @cloud_id,
-            disable_api_termination: {:value => true}
+          instance_id: @cloud_id,
+          disable_api_termination: { value: true}
         )
 
-        has_elastic_ip = false
-        if !instance.public_ip_address.nil?
-          begin
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(public_ips: [instance.public_ip_address])
-            if resp.addresses.size > 0 and resp.addresses.first.instance_id == @cloud_id
-              has_elastic_ip = true
-            end
-          rescue Aws::EC2::Errors::InvalidAddressNotFound
-            # XXX this is ok to ignore, it means the public IP isn't Elastic
-          end
-        end
-
-        win_admin_password = nil
-        ec2config_password = nil
-        sshd_password = nil
-        if windows?
-          if @config['use_cloud_provider_windows_password']
-            win_admin_password = getWindowsAdminPassword
-          elsif @config['windows_auth_vault'] && !@config['windows_auth_vault'].empty?
-            if @config["windows_auth_vault"].has_key?("password_field")
-              win_admin_password = @groomer.getSecret(
-                  vault: @config['windows_auth_vault']['vault'],
-                  item: @config['windows_auth_vault']['item'],
-                  field: @config["windows_auth_vault"]["password_field"]
-              )
-            else
-              win_admin_password = getWindowsAdminPassword
-            end
-
-            if @config["windows_auth_vault"].has_key?("ec2config_password_field")
-              ec2config_password = @groomer.getSecret(
-                  vault: @config['windows_auth_vault']['vault'],
-                  item: @config['windows_auth_vault']['item'],
-                  field: @config["windows_auth_vault"]["ec2config_password_field"]
-              )
-            end
-
-            if @config["windows_auth_vault"].has_key?("sshd_password_field")
-              sshd_password = @groomer.getSecret(
-                  vault: @config['windows_auth_vault']['vault'],
-                  item: @config['windows_auth_vault']['item'],
-                  field: @config["windows_auth_vault"]["sshd_password_field"]
-              )
-            end
-          end
-
-          win_admin_password = MU.generateWindowsPassword if win_admin_password.nil?
-          ec2config_password = MU.generateWindowsPassword if ec2config_password.nil?
-          sshd_password = MU.generateWindowsPassword if sshd_password.nil?
-
-          # We're creating the vault here so when we run
-          # MU::Cloud::Server.initialSSHTasks and we need to set the Windows
-          # Admin password we can grab it from said vault.
-          creds = {
-              "username" => @config['windows_admin_username'],
-              "password" => win_admin_password,
-              "ec2config_username" => "ec2config",
-              "ec2config_password" => ec2config_password,
-              "sshd_username" => "sshd_service",
-              "sshd_password" => sshd_password
-          }
-          @groomer.saveSecret(vault: @mu_name, item: "windows_credentials", data: creds, permissions: "name:#{@mu_name}")
-        end
-
-        subnet = nil
-        if !@vpc.nil? and @config.has_key?("vpc") and !instance.subnet_id.nil?
-          subnet = @vpc.getSubnet(
-            cloud_id: instance.subnet_id
-          )
-          if subnet.nil?
-            raise MuError, "Got null subnet id out of #{@config['vpc']} when asking for #{instance.subnet_id}"
-          end
-        end
-
-        if !subnet.nil?
-          if !subnet.private? or (!@config['static_ip'].nil? and !@config['static_ip']['assign_ip'].nil?)
-            if !@config['static_ip'].nil?
-              if !@config['static_ip']['ip'].nil?
-                MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id, classic: false, ip: @config['static_ip']['ip'])
-              elsif !has_elastic_ip
-                MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id)
-              end
-            end
-          end
-
-          _nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
-          if subnet.private? and !nat_ssh_host and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
-            raise MuError, "#{node} is in a private subnet (#{subnet}), but has no bastion host configured, and I have no other route to it"
-          end
-
-          # If we've asked for additional subnets (and this @config is not a
-          # member of a Server Pool, which has different semantics), create
-          # extra interfaces to accomodate.
-          if !@config['vpc']['subnets'].nil? and @config['basis'].nil?
-            device_index = 1
-            @vpc.subnets.each { |s|
-              subnet_id = s.cloud_id
-              MU.log "Adding network interface on subnet #{subnet_id} for #{node}"
-              iface = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_network_interface(subnet_id: subnet_id).network_interface
-              MU::Cloud::AWS.createStandardTags(iface.network_interface_id, region: @config['region'], credentials: @config['credentials'])
-              MU::Cloud::AWS.createTag(iface.network_interface_id, "Name", node+"-ETH"+device_index.to_s, region: @config['region'], credentials: @config['credentials'])
-
-              if @config['optional_tags']
-                MU::MommaCat.listOptionalTags.each { |key, value|
-                  MU::Cloud::AWS.createTag(iface.network_interface_id, key, value, region: @config['region'], credentials: @config['credentials'])
-                }
-              end
-
-              if !@config['tags'].nil?
-                @config['tags'].each { |tag|
-                  MU::Cloud::AWS.createTag(iface.network_interface_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
-                }
-              end
-
-              MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_network_interface(
-                  network_interface_id: iface.network_interface_id,
-                  instance_id: instance.instance_id,
-                  device_index: device_index
-              )
-              device_index = device_index + 1
-            }
-          end
-        elsif !@config['static_ip'].nil?
-          if !@config['static_ip']['ip'].nil?
-            MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id, classic: true, ip: @config['static_ip']['ip'])
-          elsif !has_elastic_ip
-            MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id, classic: true)
-          end
-        end
-
+        tagVolumes
+        configureNetworking
+        saveCredentials
 
         if !@config['image_then_destroy']
           notify
         end
 
-        MU.log "EC2 instance #{node} has id #{instance.instance_id}", MU::DEBUG
-
-        @config["private_dns_name"] = instance.private_dns_name
-        @config["public_dns_name"] = instance.public_dns_name
-        @config["private_ip_address"] = instance.private_ip_address
-        @config["public_ip_address"] = instance.public_ip_address
-
-        # Root disk on standard CentOS AMI
-        # tagVolumes(instance.instance_id, "/dev/sda", "Name", "ROOT-"+MU.deploy_id+"-"+@config["name"].upcase)
-        # Root disk on standard Ubuntu AMI
-        # tagVolumes(instance.instance_id, "/dev/sda1", "Name", "ROOT-"+MU.deploy_id+"-"+@config["name"].upcase)
-
-        # Generic deploy ID tag
-        # tagVolumes(instance.instance_id)
-
-        # Tag volumes with all our standard tags.
-        # Maybe replace tagVolumes with this? There is one more place tagVolumes is called from
-        volumes = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(filters: [name: "attachment.instance-id", values: [instance.instance_id]])
-        volumes.each { |vol|
-          vol.volumes.each { |volume|
-            volume.attachments.each { |attachment|
-              MU::MommaCat.listStandardTags.each_pair { |key, value|
-                MU::Cloud::AWS.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
-
-                if attachment.device == "/dev/sda" or attachment.device == "/dev/sda1"
-                  MU::Cloud::AWS.createTag(attachment.volume_id, "Name", "ROOT-#{MU.deploy_id}-#{@config["name"].upcase}", region: @config['region'], credentials: @config['credentials'])
-                else
-                  MU::Cloud::AWS.createTag(attachment.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{attachment.device.upcase}", region: @config['region'], credentials: @config['credentials'])
-                end
-              }
-
-              if @config['optional_tags']
-                MU::MommaCat.listOptionalTags.each { |key, value|
-                  MU::Cloud::AWS.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
-                }
-              end
-
-              if @config['tags']
-                @config['tags'].each { |tag|
-                  MU::Cloud::AWS.createTag(attachment.volume_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
-                }
-              end
-            }
-          }
-        }
-
-        canonical_name = instance.public_dns_name
-        canonical_name = instance.private_dns_name if !canonical_name or nat_ssh_host != nil
-        @config['canonical_name'] = canonical_name
-
-        if !@config['add_private_ips'].nil?
-          instance.network_interfaces.each { |int|
-            if int.private_ip_address == instance.private_ip_address and int.private_ip_addresses.size < (@config['add_private_ips'] + 1)
-              MU.log "Adding #{@config['add_private_ips']} extra private IP addresses to #{instance.instance_id}"
-              MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).assign_private_ip_addresses(
-                  network_interface_id: int.network_interface_id,
-                  secondary_private_ip_address_count: @config['add_private_ips'],
-                  allow_reassignment: false
-              )
-            end
-          }
-          notify
-        end
-
-        begin
-          if @config['groom'].nil? or @config['groom']
-            if windows?
-              # kick off certificate generation early; WinRM will need it
-              @deploy.nodeSSLCerts(self)
-              if @config.has_key?("basis")
-                @deploy.nodeSSLCerts(self, true)
-              end
-              if !@groomer.haveBootstrapped?
-                session = getWinRMSession(50, 60, reboot_on_problems: true)
-                initialWinRMTasks(session)
-                begin
-                  session.close
-                rescue StandardError
-                  # this is allowed to fail- we're probably rebooting anyway
-                end
-              else # for an existing Windows node: WinRM, then SSH if it fails
-                begin
-                  session = getWinRMSession(1, 60)
-                rescue StandardError # yeah, yeah
-                  session = getSSHSession(1, 60)
-                  # XXX maybe loop at least once if this also fails?
-                end
-              end
-            else
-              session = getSSHSession(40, 30)
-              initialSSHTasks(session)
-            end
-          end
-        rescue BootstrapTempFail
-          sleep 45
-          retry
-        ensure
-          session.close if !session.nil? and !windows?
-        end
-
-        if @config["existing_deploys"] && !@config["existing_deploys"].empty?
-          @config["existing_deploys"].each { |ext_deploy|
-            if ext_deploy["cloud_id"]
-              found = MU::MommaCat.findStray(
-                @config['cloud'],
-                ext_deploy["cloud_type"],
-                cloud_id: ext_deploy["cloud_id"],
-                region: @config['region'],
-                dummy_ok: false
-              ).first
-
-              MU.log "Couldn't find existing resource #{ext_deploy["cloud_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
-              @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: found.mu_name, triggering_node: @mu_name)
-            elsif ext_deploy["mu_name"] && ext_deploy["deploy_id"]
-              MU.log "#{ext_deploy["mu_name"]} / #{ext_deploy["deploy_id"]}"
-              found = MU::MommaCat.findStray(
-                @config['cloud'],
-                ext_deploy["cloud_type"],
-                deploy_id: ext_deploy["deploy_id"],
-                mu_name: ext_deploy["mu_name"],
-                region: @config['region'],
-                dummy_ok: false
-              ).first
-
-              MU.log "Couldn't find existing resource #{ext_deploy["mu_name"]}/#{ext_deploy["deploy_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
-              @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: ext_deploy["mu_name"], triggering_node: @mu_name)
-            else
-              MU.log "Trying to find existing deploy, but either the cloud_id is not valid or no mu_name and deploy_id where provided", MU::ERR
-            end
-          }
-        end
-
-        # See if this node already exists in our config management. If it does,
-        # we're done.
-        if MU.inGem?
-          MU.log "Deploying from a gem, not grooming"
-          MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
-          MU::MommaCat.unlock(instance.instance_id+"-groom")
-
-          return true
-        elsif @groomer.haveBootstrapped?
-          MU.log "Node #{node} has already been bootstrapped, skipping groomer setup.", MU::NOTICE
-
-          if @config['groom'].nil? or @config['groom']
-            @groomer.saveDeployData
-          end
-
-          MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
-          MU::MommaCat.unlock(instance.instance_id+"-groom")
-          return true
-        end
-
-        begin
-          @groomer.bootstrap if @config['groom'].nil? or @config['groom']
-        rescue MU::Groomer::RunError
-          MU::MommaCat.unlock(instance.instance_id+"-groom")
-          MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
-          return false
-        end
+        getIAMProfile
+        finish.call(false) if !bootstrapGroomer
 
         # Make sure we got our name written everywhere applicable
         if !@named
@@ -963,140 +526,75 @@ module MU
           @named = true
         end
 
-        MU::MommaCat.unlock(instance.instance_id+"-groom")
-        MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
-        return true
-      end
-
-      # postBoot
+        finish.call(true)
+      end #postboot 
 
       # Locate an existing instance or instances and return an array containing matching AWS resource descriptors for those that match.
       # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching instances
       def self.find(**args)
         ip ||= args[:flags]['ip'] if args[:flags] and args[:flags]['ip']
 
-        if !args[:region].nil?
-          regions = [args[:region]]
-        else
-          regions = MU::Cloud::AWS.listRegions
-        end
+        regions = args[:region].nil? ? MU::Cloud::AWS.listRegions : [args[:region]]
 
         found = {}
         search_semaphore = Mutex.new
         search_threads = []
 
-        if !ip and !args[:cloud_id] and !args[:tag_value]
-          regions.each { |r|
-            search_threads << Thread.new {
-              MU::Cloud::AWS.ec2(region: r, credentials: args[:credentials]).describe_instances(
-                filters: [
-                  {
-                    name: "instance-state-name",
-                    values: ["running", "pending", "stopped"]
-                  }
-                ]
-              ).reservations.each { |resp|
-                if !resp.nil? and !resp.instances.nil?
-                  resp.instances.each { |i|
-                    search_semaphore.synchronize {
-                      found[i.instance_id] = i
-                    }
-                  }
-                end
-              }
-            }
-          }
+        base_filter = { name: "instance-state-name", values: ["running", "pending", "stopped"] }
+        searches = []
 
-          search_threads.each { |t|
-            t.join
+        if args[:cloud_id]
+          searches << {
+            :instance_ids => [args[:cloud_id]],
+            :filters => [base_filter]
           }
-
-          return found
         end
 
-        # If we got an instance id, go get it
-        if args[:cloud_id]
-          regions.each { |r|
-            search_threads << Thread.new {
-              MU.log "Hunting for instance with cloud id '#{args[:cloud_id]}' in #{r}", MU::DEBUG
-              retries = 0
-              begin
-                MU::Cloud::AWS.ec2(region: r, credentials: args[:credentials]).describe_instances(
-                  instance_ids: [args[:cloud_id]],
-                  filters: [
-                    {
-                      name: "instance-state-name",
-                      values: ["running", "pending", "stopped"]
-                    }
-                  ]
-                ).reservations.each { |resp|
-                  if !resp.nil? and !resp.instances.nil?
-                    resp.instances.each { |i|
-                      search_semaphore.synchronize {
-                        found[i.instance_id] = i
-                      }
-                    }
-                  end
-                }
-              rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
-                retries += 1
-                if retries <= 5
-                  sleep 5
-                else
-                  raise MuError, "#{e.inspect} in region #{r}"
-                end
-              end
+        if ip
+          ["ip-address", "private-ip-address"].each { |ip_type|
+            searches << {
+              filters: [base_filter,  {name: ip_type, values: [ip]} ],
             }
           }
-          done_threads = []
-          begin
-            search_threads.each { |t|
-              joined = t.join(2)
-              done_threads << joined if !joined.nil?
-            }
-          end while found.size < 1 and done_threads.size != search_threads.size
         end
 
-        return found if found.size > 0
-
-        # Ok, well, let's try looking it up by IP then
-        if !ip.nil?
-          MU.log "Hunting for instance by IP '#{ip}'", MU::DEBUG
-          ["ip-address", "private-ip-address"].each { |filter|
-            regions.each { |r|
-              response = MU::Cloud::AWS.ec2(region: r, credentials: args[:credentials]).describe_instances(
-                filters: [
-                  {name: filter, values: [ip]},
-                  {name: "instance-state-name", values: ["running", "pending", "stopped"]}
-                ]
-              ).reservations.first
-              response.instances.each { |i|
-                found[i.instance_id] = i
-              }
-            }
+        if args[:tag_value] and args[:tag_key]
+          searches << {
+            filters: [
+              base_filter,
+              {name: ip_type, values: [ip]},
+              {name: "tag:#{args[:tag_key]}", values: [args[:tag_value]]},
+            ]
           }
         end
 
-        return found if found.size > 0
+        if searches.empty?
+          searches << { filters: [base_filter] }
+        end
 
-        # Fine, let's try it by tag.
-        if args[:tag_value]
-          MU.log "Searching for instance by tag '#{args[:tag_key]}=#{args[:tag_value]}'", MU::DEBUG
-          regions.each { |r|
-            MU::Cloud::AWS.ec2(region: r, credentials: args[:credentials]).describe_instances(
-              filters: [
-                {name: "tag:#{args[:tag_key]}", values: [args[:tag_value]]},
-                {name: "instance-state-name", values: ["running", "pending", "stopped"]}
-              ]
-            ).reservations.each { |resp|
-              if !resp.nil? and resp.instances.size > 0
-                resp.instances.each { |i|
-                  found[i.instance_id] = i
+        regions.each { |r|
+          searches.each { |search|
+            search_threads << Thread.new(search) { |params|
+              MU.retrier([Aws::EC2::Errors::InvalidInstanceIDNotFound], wait: 5, max: 5, ignoreme: [Aws::EC2::Errors::InvalidInstanceIDNotFound]) {
+                MU::Cloud::AWS.ec2(region: r, credentials: args[:credentials]).describe_instances(params).reservations.each { |resp|
+                  next if resp.nil? or resp.instances.nil?
+                  resp.instances.each { |i|
+                    search_semaphore.synchronize {
+                      found[i.instance_id] = i
+                    }
+                  }
                 }
-              end
+              }
             }
           }
-        end
+        }
+        done_threads = []
+        begin
+          search_threads.each { |t|
+            joined = t.join(2)
+            done_threads << joined if !joined.nil?
+          }
+        end while found.size < 1 and done_threads.size != search_threads.size
 
         return found
       end
@@ -1211,8 +709,8 @@ module MU
 
           int.private_ip_addresses.each { |priv_ip|
             if !priv_ip.primary
-              bok['add_private_ips'] ||= []
-              bok['add_private_ips'] << priv_ip.private_ip_address
+              bok['add_private_ips'] ||= 0
+              bok['add_private_ips'] += 1
             end
             if priv_ip.association and priv_ip.association.public_ip 
               bok['associate_public_ip'] = true
@@ -1345,11 +843,6 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def groom
           MU::MommaCat.lock(@cloud_id+"-groom")
-          node, _config, deploydata = describe(cloud_id: @cloud_id)
-
-          if node.nil? or node.empty?
-            raise MuError, "MU::Cloud::AWS::Server.groom was called without a mu_name"
-          end
 
           # Make double sure we don't lose a cached mu_windows_name value.
           if windows? or !@config['active_directory'].nil?
@@ -1360,7 +853,7 @@ module MU
 
           punchAdminNAT
 
-          MU::Cloud::AWS::Server.tagVolumes(@cloud_id, credentials: @config['credentials'])
+          tagVolumes
 
           # If we have a loadbalancer configured, attach us to it
           if !@config['loadbalancers'].nil?
@@ -1389,55 +882,18 @@ module MU
           end
 
           begin
+            getIAMProfile
             if @config['groom'].nil? or @config['groom']
-              @groomer.run(purpose: "Full Initial Run", max_retries: 15, reboot_first_fail: windows?, timeout: @config['groomer_timeout'])
+              @groomer.run(purpose: "Full Initial Run", max_retries: 15, reboot_first_fail: (windows? and @config['groomer'] != "Ansible"), timeout: @config['groomer_timeout'])
             end
           rescue MU::Groomer::RunError => e
-            MU.log "Proceeding after failed initial Groomer run, but #{node} may not behave as expected!", MU::WARN, details: e.message
+            MU.log "Proceeding after failed initial Groomer run, but #{@mu_name} may not behave as expected!", MU::WARN, details: e.message
           rescue StandardError => e
-            MU.log "Caught #{e.inspect} on #{node} in an unexpected place (after @groomer.run on Full Initial Run)", MU::ERR
+            MU.log "Caught #{e.inspect} on #{@mu_name} in an unexpected place (after @groomer.run on Full Initial Run)", MU::ERR
           end
 
           if !@config['create_image'].nil? and !@config['image_created']
-            img_cfg = @config['create_image']
-            # Scrub things that don't belong on an AMI
-            session = getSSHSession
-            sudo = purgecmd = ""
-            sudo = "sudo" if @config['ssh_user'] != "root"
-            if windows?
-              purgecmd = "rm -rf /cygdrive/c/mu_installed_chef"
-            else
-              purgecmd = "rm -rf /opt/mu_installed_chef"
-            end
-            if img_cfg['image_then_destroy']
-              if windows?
-                purgecmd = "rm -rf /cygdrive/c/chef/ /home/#{@config['windows_admin_username']}/.ssh/authorized_keys /home/Administrator/.ssh/authorized_keys /cygdrive/c/mu-installer-ran-updates /cygdrive/c/mu_installed_chef"
-                # session.exec!("powershell -Command \"& {(Get-WmiObject -Class Win32_Product -Filter \"Name='UniversalForwarder'\").Uninstall()}\"")
-              else
-                purgecmd = "#{sudo} rm -rf /var/lib/cloud/instances/i-* /root/.ssh/authorized_keys /etc/ssh/ssh_host_*key* /etc/chef /etc/opscode/* /.mu-installer-ran-updates /var/chef /opt/mu_installed_chef /opt/chef ; #{sudo} sed -i 's/^HOSTNAME=.*//' /etc/sysconfig/network"
-              end
-            end
-            session.exec!(purgecmd)
-            session.close
-            ami_ids = MU::Cloud::AWS::Server.createImage(
-                name: @mu_name,
-                instance_id: @cloud_id,
-                storage: @config['storage'],
-                exclude_storage: img_cfg['image_exclude_storage'],
-                copy_to_regions: img_cfg['copy_to_regions'],
-                make_public: img_cfg['public'],
-                region: @config['region'],
-                tags: @config['tags'],
-                credentials: @config['credentials']
-            )
-            @deploy.notify("images", @config['name'], ami_ids)
-            @config['image_created'] = true
-            if img_cfg['image_then_destroy']
-              MU::Cloud::AWS::Server.waitForAMI(ami_ids[@config['region']], region: @config['region'], credentials: @config['credentials'])
-              MU.log "AMI #{ami_ids[@config['region']]} ready, removing source node #{node}"
-              MU::Cloud::AWS::Server.terminateInstance(id: @cloud_id, region: @config['region'], deploy_id: @deploy.deploy_id, mu_name: @mu_name, credentials: @config['credentials'])
-              destroy
-            end
+            createImage
           end
 
           MU::MommaCat.unlock(@cloud_id+"-groom")
@@ -1485,23 +941,19 @@ module MU
         # bastion hosts that may be in the path, see getSSHConfig if that's what
         # you need.
         def canonicalIP
-          _mu_name, _config, deploydata = describe(cloud_id: @cloud_id)
-
-          instance = cloud_desc
-
-          if !instance
+          if !cloud_desc
             raise MuError, "Couldn't retrieve cloud descriptor for server #{self}"
           end
 
           if deploydata.nil? or
               (!deploydata.has_key?("private_ip_address") and
                   !deploydata.has_key?("public_ip_address"))
-            return nil if instance.nil?
+            return nil if cloud_desc.nil?
             @deploydata = {} if @deploydata.nil?
-            @deploydata["public_ip_address"] = instance.public_ip_address
-            @deploydata["public_dns_name"] = instance.public_dns_name
-            @deploydata["private_ip_address"] = instance.private_ip_address
-            @deploydata["private_dns_name"] = instance.private_dns_name
+            @deploydata["public_ip_address"] = cloud_desc.public_ip_address
+            @deploydata["public_dns_name"] = cloud_desc.public_dns_name
+            @deploydata["private_ip_address"] = cloud_desc.private_ip_address
+            @deploydata["private_dns_name"] = cloud_desc.private_dns_name
 
             notify
           end
@@ -1510,13 +962,13 @@ module MU
           # which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
           # The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
           if MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials']) or @deploydata["public_ip_address"].nil?
-            @config['canonical_ip'] = instance.private_ip_address
-            @deploydata["private_ip_address"] = instance.private_ip_address
-            return instance.private_ip_address
+            @config['canonical_ip'] = cloud_desc.private_ip_address
+            @deploydata["private_ip_address"] = cloud_desc.private_ip_address
+            return cloud_desc.private_ip_address
           else
-            @config['canonical_ip'] = instance.public_ip_address
-            @deploydata["public_ip_address"] = instance.public_ip_address
-            return instance.public_ip_address
+            @config['canonical_ip'] = cloud_desc.public_ip_address
+            @deploydata["public_ip_address"] = cloud_desc.public_ip_address
+            return cloud_desc.public_ip_address
           end
         end
 
@@ -1709,7 +1161,26 @@ module MU
         # Retrieves the Cloud provider's randomly generated Windows password
         # Will only work on stock Amazon Windows AMIs or custom AMIs that where created with Administrator Password set to random in EC2Config
         # return [String]: A password string.
-        def getWindowsAdminPassword
+        def getWindowsAdminPassword(use_cache: true)
+          @config['windows_auth_vault'] ||= {
+            "vault" => @mu_name,
+            "item" => "windows_credentials",
+            "password_field" => "password"
+          }
+
+          if use_cache
+            begin
+              win_admin_password = @groomer.getSecret(
+                vault: @config['windows_auth_vault']['vault'],
+                item: @config['windows_auth_vault']['item'],
+                field: @config["windows_auth_vault"]["password_field"]
+              )
+
+              return win_admin_password if win_admin_password
+            rescue MU::Groomer::MuNoSuchSecret, MU::Groomer::RunError
+            end
+          end
+
           if @cloud_id.nil?
             describe
             @cloud_id = cloud_desc.instance_id
@@ -1748,6 +1219,8 @@ module MU
           pem_bytes = File.open("#{ssh_keydir}/#{ssh_key_name}", 'rb') { |f| f.read }
           private_key = OpenSSL::PKey::RSA.new(pem_bytes)
           decrypted_password = private_key.private_decrypt(decoded)
+          saveCredentials(decrypted_password)
+
           return decrypted_password
         end
 
@@ -1821,60 +1294,37 @@ module MU
         # @param type [String]: Cloud storage type of the volume, if applicable
         # @param delete_on_termination [Boolean]: Value of delete_on_termination flag to set
         def addVolume(dev, size, type: "gp2", delete_on_termination: false)
-          if @cloud_id.nil? or @cloud_id.empty?
-            MU.log "#{self} didn't have a cloud id, couldn't determine 'active?' status", MU::ERR
-            return true
+
+          if setDeleteOntermination(dev, delete_on_termination)
+            MU.log "A volume #{device} already attached to #{self}, skipping", MU::NOTICE
+            return
           end
-          az = nil
-          MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(
-            instance_ids: [@cloud_id]
-          ).reservations.each { |resp|
-            if !resp.nil? and !resp.instances.nil?
-              resp.instances.each { |instance|
-                az = instance.placement.availability_zone
-                mappings = MU.structToHash(instance.block_device_mappings)
-                mappings.each { |vol|
-                  if vol[:ebs]
-                    vol[:ebs].delete(:attach_time)
-                    vol[:ebs].delete(:status)
-                  end
-                }
-                mappings.each { |vol|
-                  if vol[:device_name] == dev
-                    MU.log "A volume #{dev} already attached to #{self}, skipping", MU::NOTICE
-                    if vol[:ebs][:delete_on_termination] != delete_on_termination
-                      vol[:ebs][:delete_on_termination] = delete_on_termination
-                      MU.log "Setting delete_on_termination flag to #{delete_on_termination.to_s} on #{@mu_name}'s #{dev}"
-                      MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
-                        instance_id: @cloud_id,
-                        block_device_mappings: mappings
-                      )
-                    end
-                    return
-                  end
-                }
-              }
-            end
-          }
+
           MU.log "Creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
           creation = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_volume(
-            availability_zone: az,
+            availability_zone: cloud_desc.placement.availability_zone,
             size: size,
             volume_type: type
           )
-          begin
-            sleep 3
+
+          MU.retrier(wait: 3, loop_if: Proc.new {
             creation = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(volume_ids: [creation.volume_id]).volumes.first
             if !["creating", "available"].include?(creation.state)
               raise MuError, "Saw state '#{creation.state}' while creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
             end
-          end while creation.state != "available"
+            creation.state != "available"
+          })
+
 
           if @deploy
-            MU::MommaCat.listStandardTags.each_pair { |key, value|
-              MU::Cloud::AWS.createTag(creation.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
-            }
-            MU::Cloud::AWS.createTag(creation.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{dev.upcase}", region: @config['region'], credentials: @config['credentials'])
+            MU::Cloud::AWS.createStandardTags(
+              resource_id,
+              region: @config['region'],
+              credentials: @config['credentials'],
+              optional: @config['optional_tags'],
+              nametag: @mu_name+"-"+dev.upcase,
+              othertags: @config['tags']
+            )
           end
 
           attachment = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_volume(
@@ -1893,29 +1343,7 @@ module MU
 
           # Set delete_on_termination, which for some reason is an instance
           # attribute and not on the attachment
-          mappings = MU.structToHash(cloud_desc.block_device_mappings)
-          changed = false
-
-          mappings.each { |mapping|
-            if mapping[:ebs]
-              mapping[:ebs].delete(:attach_time)
-              mapping[:ebs].delete(:status)
-            end
-            if mapping[:device_name] == dev and 
-               mapping[:ebs][:delete_on_termination] != delete_on_termination
-              changed = true
-              mapping[:ebs][:delete_on_termination] = delete_on_termination
-            end
-          }
-
-          if changed
-            MU.log "Setting delete_on_termination flag to #{delete_on_termination.to_s} on #{@mu_name}'s #{dev}"
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
-              instance_id: @cloud_id,
-              block_device_mappings: mappings
-            )
-          end
-
+          setDeleteOntermination(dev, delete_on_termination)
         end
 
         # Determine whether the node in question exists at the Cloud provider
@@ -1953,13 +1381,13 @@ module MU
         # @param ip [String]: Request a specific IP address.
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.associateElasticIp(instance_id, classic: false, ip: nil, region: MU.curRegion)
+        def self.associateElasticIp(instance_id, classic: false, ip: nil, region: MU.curRegion, credentials: nil)
           MU.log "associateElasticIp called: #{instance_id}, classic: #{classic}, ip: #{ip}, region: #{region}", MU::DEBUG
           elastic_ip = nil
           @eip_semaphore.synchronize {
             if !ip.nil?
               filters = [{name: "public-ip", values: [ip]}]
-              resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(filters: filters)
+              resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_addresses(filters: filters)
               if @eips_used.include?(ip)
                 is_free = false
                 resp.addresses.each { |address|
@@ -1988,54 +1416,44 @@ module MU
             @eips_used << elastic_ip.public_ip
             MU.log "Associating Elastic IP #{elastic_ip.public_ip} with #{instance_id}", details: elastic_ip
           }
-          attempts = 0
-          begin
+
+          on_retry = Proc.new { |e|
+            if e.class == Aws::EC2::Errors::ResourceAlreadyAssociated
+              # A previous association attempt may have succeeded, albeit slowly.
+              resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_addresses(
+                allocation_ids: [elastic_ip.allocation_id]
+              )
+              first_addr = resp.addresses.first
+              if first_addr and first_addr.instance_id != instance_id
+                raise MuError, "Tried to associate #{elastic_ip.public_ip} with #{instance_id}, but it's already associated with #{first_addr.instance_id}!"
+              end
+            end
+          }
+
+          MU.retrier([Aws::EC2::Errors::IncorrectInstanceState, Aws::EC2::Errors::ResourceAlreadyAssociated], wait: 5, max: 6, on_retry: on_retry) {
             if classic
-              resp = MU::Cloud::AWS.ec2(region: region).associate_address(
-                  instance_id: instance_id,
-                  public_ip: elastic_ip.public_ip
+              MU::Cloud::AWS.ec2(region: region, credentials: credentials).associate_address(
+                instance_id: instance_id,
+                public_ip: elastic_ip.public_ip
               )
             else
-              resp = MU::Cloud::AWS.ec2(region: region).associate_address(
-                  instance_id: instance_id,
-                  allocation_id: elastic_ip.allocation_id,
-                  allow_reassociation: false
+              MU::Cloud::AWS.ec2(region: region, credentials: credentials).associate_address(
+                instance_id: instance_id,
+                allocation_id: elastic_ip.allocation_id,
+                allow_reassociation: false
               )
             end
-          rescue Aws::EC2::Errors::IncorrectInstanceState => e
-            attempts = attempts + 1
-            if attempts < 6
-              MU.log "Got #{e.message} associating #{elastic_ip.allocation_id} with #{instance_id}, retrying", MU::WARN
-              sleep 5
-              retry
-            end
-            raise MuError "#{e.message} associating #{elastic_ip.allocation_id} with #{instance_id}"
-          rescue Aws::EC2::Errors::ResourceAlreadyAssociated => e
-            # A previous association attempt may have succeeded, albeit slowly.
-            resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(
-                allocation_ids: [elastic_ip.allocation_id]
-            )
-            first_addr = resp.addresses.first
-            if !first_addr.nil? and first_addr.instance_id == instance_id
-              MU.log "#{elastic_ip.public_ip} already associated with #{instance_id}", MU::WARN
-            else
-              MU.log "#{elastic_ip.public_ip} shows as already associated!", MU::ERR, details: resp
-              raise MuError, "#{elastic_ip.public_ip} shows as already associated with #{first_addr.instance_id}!"
-            end
-          end
+          }
 
-          instance = MU::Cloud::AWS.ec2(region: region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
-          waited = false
-          if instance.public_ip_address != elastic_ip.public_ip
-            waited = true
-            begin
-              sleep 10
-              MU.log "Waiting for Elastic IP association of #{elastic_ip.public_ip} to #{instance_id} to take effect", MU::NOTICE
-              instance = MU::Cloud::AWS.ec2(region: region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
-            end while instance.public_ip_address != elastic_ip.public_ip
-          end
+          loop_if = Proc.new {
+            instance = find(cloud_id: instance_id, region: region, credentials: credentials).values.first
+            instance.public_ip_address != elastic_ip.public_ip
+          }
+          MU.retrier(loop_if: loop_if, wait: 10, max: 3) {
+            MU.log "Waiting for Elastic IP association of #{elastic_ip.public_ip} to #{instance_id} to take effect", MU::NOTICE
+          }
 
-          MU.log "Elastic IP #{elastic_ip.public_ip} now associated with #{instance_id}" if waited
+          MU.log "Elastic IP #{elastic_ip.public_ip} now associated with #{instance_id}"
 
           return elastic_ip.public_ip
         end
@@ -2117,193 +1535,105 @@ module MU
           }
         end
 
+        # Return an instance's AWS-assigned IP addresses and hostnames.
+        # @param instance [OpenStruct]
+        # @param id [String]
+        # @param region [String]
+        # @param credentials [@String]
+        # @return [Array<Array>]
+        def self.getAddresses(instance = nil, id: nil, region: MU.curRegion, credentials: nil)
+          return nil if !instance and !id
+
+          instance ||= find(cloud_id: id, region: region, credentials: credentials).values.first
+          return if !instance
+
+          ips = []
+          names = []
+          instance.network_interfaces.each { |iface|
+            iface.private_ip_addresses.each { |ip|
+              ips << ip.private_ip_address
+              names << ip.private_dns_name
+              if ip.association
+                ips << ip.association.public_ip
+                names << ip.association.public_dns_name
+              end
+            }
+          }
+
+          [ips, names]
+        end
+
         # Terminate an instance.
         # @param instance [OpenStruct]: The cloud provider's description of the instance.
         # @param id [String]: The cloud provider's identifier for the instance, to use if the full description is not available.
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.terminateInstance(instance: nil, noop: false, id: nil, onlycloud: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, credentials: nil)
-          ips = Array.new
-          if !instance
-            if id
-              begin
-                resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
-              rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
-                MU.log "Instance #{id} no longer exists", MU::WARN
-              end
-              if !resp.nil? and !resp.reservations.nil? and !resp.reservations.first.nil?
-                instance = resp.reservations.first.instances.first
-                ips << instance.public_ip_address if !instance.public_ip_address.nil?
-                ips << instance.private_ip_address if !instance.private_ip_address.nil?
-              end
-            else
-              MU.log "You must supply an instance handle or id to terminateInstance", MU::ERR
-            end
-          else
-            id = instance.instance_id
-          end
-          if !MU.deploy_id.empty?
-            deploy_dir = File.expand_path("#{MU.dataDir}/deployments/"+MU.deploy_id)
-            if Dir.exist?(deploy_dir) and !noop
-              FileUtils.touch("#{deploy_dir}/.cleanup-"+id)
-            end
+          if !id and !instance
+            MU.log "You must supply an instance handle or id to terminateInstance", MU::ERR
+            return
           end
+          instance ||= find(cloud_id: id, region: region, credentials: credentials).values.first
+          return if !instance
 
-          server_obj = MU::MommaCat.findStray(
-              "AWS",
-              "servers",
-              region: region,
-              deploy_id: deploy_id,
-              cloud_id: id,
-              mu_name: mu_name
-          ).first
+          id ||= instance.instance_id
+          MU::MommaCat.lock(".cleanup-"+id)
 
-          begin
-            MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
-          rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
-            MU.log "Instance #{id} no longer exists", MU::DEBUG
-          end
-
-          if !server_obj.nil? and MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud?
-            # DNS cleanup is now done in MU::Cloud::DNSZone. Keeping this for now
-            cleaned_dns = false
-            mu_name = server_obj.mu_name
-            mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: credentials).values.first
-            if !mu_zone.nil?
-              zone_rrsets = []
-              rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: mu_zone.id)
-              rrsets.resource_record_sets.each{ |record|
-                zone_rrsets << record
-              }
+          ips, names = getAddresses(instance, region: region, credentials: credentials)
+          targets = ips +names
 
-            # AWS API returns a maximum of 100 results. DNS zones are likely to have more than 100 records, lets page and make sure we grab all records in a given zone
-              while rrsets.next_record_name && rrsets.next_record_type
-                rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: mu_zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
-                rrsets.resource_record_sets.each{ |record|
-                  zone_rrsets << record
-                }
-              end
-            end
-            if !onlycloud and !mu_name.nil?
-              # DNS cleanup is now done in MU::Cloud::DNSZone. Keeping this for now
-              if !zone_rrsets.nil? and !zone_rrsets.empty?
-                zone_rrsets.each { |rrset|
-                  if rrset.name.match(/^#{mu_name.downcase}\.server\.#{MU.myInstanceId}\.platform-mu/i)
-                    rrset.resource_records.each { |record|
-                      MU::Cloud::DNSZone.genericMuDNSEntry(name: mu_name, target: record.value, cloudclass: MU::Cloud::Server, delete: true)
-                      cleaned_dns = true
-                    }
-                  end
-                }
-              end
-
-							if !noop
-                if !server_obj.nil? and !server_obj.config.nil?
-			            MU.mommacat.notify(MU::Cloud::Server.cfg_plural, server_obj.config['name'], {}, mu_name: server_obj.mu_name, remove: true) if MU.mommacat
-								end
-							end
+          server_obj = MU::MommaCat.findStray(
+            "AWS",
+            "servers",
+            region: region,
+            deploy_id: deploy_id,
+            cloud_id: id,
+            mu_name: mu_name,
+            dummy_ok: true
+          ).first
 
-              # If we didn't manage to find this instance's Route53 entry by sifting
-              # deployment metadata, see if we can get it with the Name tag.
-              if !mu_zone.nil? and !cleaned_dns and !instance.nil?
-                instance.tags.each { |tag|
-                  if tag.key == "Name"
-                    zone_rrsets.each { |rrset|
-                      if rrset.name.match(/^#{tag.value.downcase}\.server\.#{MU.myInstanceId}\.platform-mu/i)
-                        rrset.resource_records.each { |record|
-                          MU::Cloud::DNSZone.genericMuDNSEntry(name: tag.value, target: record.value, cloudclass: MU::Cloud::Server, delete: true) if !noop
-                        }
-                      end
-                    }
-                  end
-                }
-              end
-            end
+          if MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud? and server_obj
+            targets.each { |target|
+              MU::Cloud::DNSZone.genericMuDNSEntry(name: server_obj.mu_name, target: target, cloudclass: MU::Cloud::Server, delete: true, noop: noop)
+            }
           end
 
-          if ips.size > 0 and !onlycloud
-            known_hosts_files = [Etc.getpwuid(Process.uid).dir+"/.ssh/known_hosts"]
-            if Etc.getpwuid(Process.uid).name == "root" and !MU.inGem?
-              begin
-                known_hosts_files << Etc.getpwnam("nagios").dir+"/.ssh/known_hosts"
-              rescue ArgumentError
-                # we're in a non-nagios environment and that's ok
-              end
-            end
-            known_hosts_files.each { |known_hosts|
-              next if !File.exist?(known_hosts)
-              MU.log "Cleaning up #{ips} from #{known_hosts}"
-              if !noop
-                File.open(known_hosts, File::CREAT|File::RDWR, 0644) { |f|
-                  f.flock(File::LOCK_EX)
-                  newlines = Array.new
-                  f.readlines.each { |line|
-                    ip_match = false
-                    ips.each { |ip|
-                      if line.match(/(^|,| )#{ip}( |,)/)
-                        MU.log "Expunging #{ip} from #{known_hosts}"
-                        ip_match = true
-                      end
-                    }
-                    newlines << line if !ip_match
-                  }
-                  f.rewind
-                  f.truncate(0)
-                  f.puts(newlines)
-                  f.flush
-                  f.flock(File::LOCK_UN)
-                }
-              end
+          if targets.size > 0 and !onlycloud
+            MU::Master.removeInstanceFromEtcHosts(server_obj.mu_name) if !noop and server_obj
+            targets.each { |target|
+              next if !target.match(/^\d+\.\d+\.\d+\.\d+$/)
+              MU::Master.removeIPFromSSHKnownHosts(target, noop: noop)
             }
           end
 
-          return if instance.nil?
+          on_retry = Proc.new {
+            instance = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [instance.instance_id]).reservations.first.instances.first
+            if instance.state.name == "terminated"
+              MU.log "#{instance.instance_id}#{server_obj ? " ("+server_obj.mu_name+")" : ""} has already been terminated, skipping"
+              MU::MommaCat.unlock(".cleanup-"+id)
+              return
+            end
+          }
 
-          name = ""
-          instance.tags.each { |tag|
-            name = tag.value if tag.key == "Name"
+          loop_if = Proc.new {
+            instance = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [instance.instance_id]).reservations.first.instances.first
+            instance.state.name != "terminated"
           }
 
-          if instance.state.name == "terminated"
-            MU.log "#{instance.instance_id} (#{name}) has already been terminated, skipping"
-          else
-            if instance.state.name == "terminating"
-              MU.log "#{instance.instance_id} (#{name}) already terminating, waiting"
-            elsif instance.state.name != "running" and instance.state.name != "pending" and instance.state.name != "stopping" and instance.state.name != "stopped"
-              MU.log "#{instance.instance_id} (#{name}) is in state #{instance.state.name}, waiting"
-            else
-              MU.log "Terminating #{instance.instance_id} (#{name}) #{noop}"
-              if !noop
-                begin
-                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_instance_attribute(
-                      instance_id: instance.instance_id,
-                      disable_api_termination: {value: false}
-                  )
-                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).terminate_instances(instance_ids: [instance.instance_id])
-                    # Small race window here with the state changing from under us
-                rescue Aws::EC2::Errors::IncorrectInstanceState => e
-                  resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
-                  if !resp.nil? and !resp.reservations.nil? and !resp.reservations.first.nil?
-                    instance = resp.reservations.first.instances.first
-                    if !instance.nil? and instance.state.name != "terminated" and instance.state.name != "terminating"
-                      sleep 5
-                      retry
-                    end
-                  end
-                rescue Aws::EC2::Errors::InternalError => e
-                  MU.log "Error #{e.inspect} while Terminating instance #{instance.instance_id} (#{name}), retrying", MU::WARN, details: e.inspect
-                  sleep 5
-                  retry
-                end
-              end
-            end
-            while instance.state.name != "terminated" and !noop
-              sleep 30
-              instance_response = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [instance.instance_id])
-              instance = instance_response.reservations.first.instances.first
-            end
-            MU.log "#{instance.instance_id} (#{name}) terminated" if !noop
+          MU.log "Terminating #{instance.instance_id}#{server_obj ? " ("+server_obj.mu_name+")" : ""}"
+          if !noop
+            MU.retrier([Aws::EC2::Errors::IncorrectInstanceState, Aws::EC2::Errors::InternalError], wait: 30, max: 60, loop_if: loop_if, on_retry: on_retry) {
+              MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_instance_attribute(
+                instance_id: instance.instance_id,
+                disable_api_termination: {value: false}
+              )
+              MU::Cloud::AWS.ec2(credentials: credentials, region: region).terminate_instances(instance_ids: [instance.instance_id])
+            }
           end
+
+          MU.log "#{instance.instance_id}#{server_obj ? " ("+server_obj.mu_name+")" : ""} terminated" if !noop
+          MU::MommaCat.unlock(".cleanup-"+id)
+
         end
 
         # Return a BoK-style config hash describing a NAT instance. We use this
@@ -2479,6 +1809,50 @@ module MU
           size
         end
 
+        # Boilerplate generation of an instance role
+        # @param server [Hash]: The BoK-style config hash for a +Server+ or +ServerPool+
+        # @param configurator [MU::Config]
+        def self.generateStandardRole(server, configurator)
+          role = {
+            "name" => server["name"],
+            "credentials" => server["credentials"],
+            "can_assume" => [
+              {
+                "entity_id" => "ec2.amazonaws.com",
+                "entity_type" => "service"
+              }
+            ],
+            "policies" => [
+              {
+                "name" => "MuSecrets",
+                "permissions" => ["s3:GetObject"],
+                "targets" => [
+                  {
+                    "identifier" => 'arn:'+(MU::Cloud::AWS.isGovCloud?(server['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(server['credentials'])+'/Mu_CA.pem'
+                  }
+                ]
+              }
+            ]
+          }
+          if server['iam_policies']
+            role['iam_policies'] = server['iam_policies'].dup
+          end
+          if server['canned_iam_policies']
+            role['import'] = server['canned_iam_policies'].dup
+          end
+          if server['iam_role']
+# XXX maybe break this down into policies and add those?
+          end
+
+          configurator.insertKitten(role, "roles")
+
+          server["dependencies"] ||= []
+          server["dependencies"] << {
+            "type" => "role",
+            "name" => server["name"]
+          }
+        end
+
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
         # @param server [Hash]: The resource to process and validate
         # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
@@ -2499,43 +1873,7 @@ module MU
               ok = false
             end
           else
-            role = {
-              "name" => server["name"],
-              "credentials" => server["credentials"],
-              "can_assume" => [
-                {
-                  "entity_id" => "ec2.amazonaws.com",
-                  "entity_type" => "service"
-                }
-              ],
-              "policies" => [
-                {
-                  "name" => "MuSecrets",
-                  "permissions" => ["s3:GetObject"],
-                  "targets" => [
-                    {
-                      "identifier" => 'arn:'+(MU::Cloud::AWS.isGovCloud?(server['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(server['credentials'])+'/Mu_CA.pem'
-                    }
-                  ]
-                }
-              ]
-            }
-            if server['iam_policies']
-              role['iam_policies'] = server['iam_policies'].dup
-            end
-            if server['canned_iam_policies']
-              role['import'] = server['canned_iam_policies'].dup
-            end
-            if server['iam_role']
-# XXX maybe break this down into policies and add those?
-            end
-
-            configurator.insertKitten(role, "roles")
-            server["dependencies"] ||= []
-            server["dependencies"] << {
-              "type" => "role",
-              "name" => server["name"]
-            }
+            generateStandardRole(server, configurator)
           end
           if !server['create_image'].nil?
             if server['create_image'].has_key?('copy_to_regions') and
@@ -2547,12 +1885,12 @@ module MU
             end
           end
 
-          server['ami_id'] ||= server['image_id']
+          server['image_id'] ||= server['ami_id']
 
-          if server['ami_id'].nil?
+          if server['image_id'].nil?
             img_id = MU::Cloud.getStockImage("AWS", platform: server['platform'], region: server['region'])
             if img_id
-              server['ami_id'] = configurator.getTail("server"+server['name']+"AMI", value: img_id, prettyname: "server"+server['name']+"AMI", cloudtype: "AWS::EC2::Image::Id")
+              server['image_id'] = configurator.getTail("server"+server['name']+"AMI", value: img_id, prettyname: "server"+server['name']+"AMI", cloudtype: "AWS::EC2::Image::Id")
             else
               MU.log "No AMI specified for #{server['name']} and no default available for platform #{server['platform']} in region #{server['region']}", MU::ERR, details: server
               ok = false
@@ -2561,22 +1899,16 @@ module MU
 
           if !server["loadbalancers"].nil?
             server["loadbalancers"].each { |lb|
-              if lb["concurrent_load_balancer"] != nil
+              lb["name"] ||= lb["concurrent_load_balancer"]
+              if lb["name"]
                 server["dependencies"] << {
-                    "type" => "loadbalancer",
-                    "name" => lb["concurrent_load_balancer"]
+                  "type" => "loadbalancer",
+                  "name" => lb["name"]
                 }
               end
             }
           end
 
-          if !server["vpc"].nil?
-            if server["vpc"]["subnet_name"].nil? and server["vpc"]["subnet_id"].nil? and server["vpc"]["subnet_pref"].nil?
-              MU.log "A server VPC block must specify a target subnet", MU::ERR
-              ok = false
-            end
-          end
-
           ok
         end
 
@@ -2604,10 +1936,11 @@ module MU
             resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_volumes(volume_ids: [volume.volume_id])
             volume = resp.data.volumes.first
           end
-          name = ""
+          name = nil
           volume.tags.each { |tag|
             name = tag.value if tag.key == "Name"
           }
+          name ||= volume.volume_id
 
           MU.log("Deleting volume #{volume.volume_id} (#{name})")
           if !noop
@@ -2630,31 +1963,405 @@ module MU
               end
             end
 
-            retries = 0
             begin
-              MU::Cloud::AWS.ec2(region: region, credentials: credentials).delete_volume(volume_id: volume.volume_id)
-            rescue Aws::EC2::Errors::IncorrectState => e
-              MU.log "Volume #{volume.volume_id} (#{name}) in incorrect state (#{e.message}), will retry", MU::WARN
-              sleep 30
-              retry
-            rescue Aws::EC2::Errors::InvalidVolumeNotFound
-              MU.log "Volume #{volume.volume_id} (#{name}) disappeared before I could remove it!", MU::WARN
+              MU.retrier([Aws::EC2::Errors::IncorrectState, Aws::EC2::Errors::VolumeInUse], ignoreme: [Aws::EC2::Errors::InvalidVolumeNotFound], wait: 30, max: 10){
+                MU::Cloud::AWS.ec2(region: region, credentials: credentials).delete_volume(volume_id: volume.volume_id)
+              }
             rescue Aws::EC2::Errors::VolumeInUse
-              if retries < 10
-                volume.attachments.each { |attachment|
-                  MU.log "#{volume.volume_id} is attached to #{attachment.instance_id} as #{attachment.device}", MU::NOTICE
-                }
-                MU.log "Volume '#{name}' is still attached, waiting...", MU::NOTICE
-                sleep 30
-                retries = retries + 1
-                retry
+              MU.log "Failed to delete #{name}", MU::ERR
+            end
+
+          end
+        end
+        private_class_method :delete_volume
+
+        # Given some combination of a base image, BoK-configured storage, and
+        # ephemeral devices, return the structure passed to EC2 to declare
+        # block devicde mappings.
+        # @param image_id [String]
+        # @param storage [Array]
+        # @param add_ephemeral [Boolean]
+        # @param region [String]
+        # @param credentials [String]
+        def self.configureBlockDevices(image_id: nil, storage: nil, add_ephemeral: true, region: MU.myRegion, credentials: nil)
+          ext_disks = {}
+  
+          # Figure out which devices are embedded in the AMI already.
+          if image_id
+            image = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_images(image_ids: [image_id]).images.first
+            if !image.block_device_mappings.nil?
+              image.block_device_mappings.each { |disk|
+                if !disk.device_name.nil? and !disk.device_name.empty? and !disk.ebs.nil? and !disk.ebs.empty?
+                  ext_disks[disk.device_name] = MU.structToHash(disk.ebs)
+                end
+              }
+            end
+          end
+  
+          configured_storage = []
+          if storage
+            storage.each { |vol|
+              # Drop the "encrypted" flag if a snapshot for this device exists
+              # in the AMI, even if they both agree about the value of said
+              # flag. Apparently that's a thing now.
+              if ext_disks.has_key?(vol["device"])
+                if ext_disks[vol["device"]].has_key?(:snapshot_id)
+                  vol.delete("encrypted")
+                end
+              end
+              mapping, _cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
+              configured_storage << mapping
+            }
+          end
+  
+          configured_storage.concat(@ephemeral_mappings) if add_ephemeral
+  
+          configured_storage
+        end
+
+        private
+
+        def bootstrapGroomer
+          if (@config['groom'].nil? or @config['groom']) and !@groomer.haveBootstrapped?
+            MU.retrier([BootstrapTempFail], wait: 45) {
+              if windows? 
+                # kick off certificate generation early; WinRM will need it
+                @deploy.nodeSSLCerts(self)
+                @deploy.nodeSSLCerts(self, true) if @config.has_key?("basis")
+                session = getWinRMSession(50, 60, reboot_on_problems: true)
+                initialWinRMTasks(session)
+                begin
+                  session.close
+                rescue StandardError
+                  # session.close is allowed to fail- we're probably rebooting
+                end
               else
-                MU.log "Failed to delete #{name}", MU::ERR
+                session = getSSHSession(40, 30)
+                initialSSHTasks(session)
+              end
+            }
+          end
+
+          # See if this node already exists in our config management. If it
+          # does, we're done.
+
+          if MU.inGem?
+            MU.log "Deploying from a gem, not grooming"
+          elsif @config['groom'].nil? or @config['groom']
+            if @groomer.haveBootstrapped?
+              MU.log "Node #{@mu_name} has already been bootstrapped, skipping groomer setup.", MU::NOTICE
+            else
+              begin
+                @groomer.bootstrap
+              rescue MU::Groomer::RunError
+                return false
               end
             end
+            @groomer.saveDeployData
+          end
+
+          true
+        end
+
+        def saveCredentials(win_admin_password = nil)
+          ec2config_password = nil
+          sshd_password = nil
+          if windows?
+            if @config['use_cloud_provider_windows_password']
+              win_admin_password ||= getWindowsAdminPassword
+            elsif @config['windows_auth_vault'] and !@config['windows_auth_vault'].empty?
+              if @config["windows_auth_vault"].has_key?("password_field")
+                win_admin_password ||= @groomer.getSecret(
+                  vault: @config['windows_auth_vault']['vault'],
+                  item: @config['windows_auth_vault']['item'],
+                  field: @config["windows_auth_vault"]["password_field"]
+                )
+              else
+                win_admin_password ||= getWindowsAdminPassword
+              end
+
+              if @config["windows_auth_vault"].has_key?("ec2config_password_field")
+                ec2config_password = @groomer.getSecret(
+                  vault: @config['windows_auth_vault']['vault'],
+                  item: @config['windows_auth_vault']['item'],
+                  field: @config["windows_auth_vault"]["ec2config_password_field"]
+                )
+              end
+
+              if @config["windows_auth_vault"].has_key?("sshd_password_field")
+                sshd_password = @groomer.getSecret(
+                  vault: @config['windows_auth_vault']['vault'],
+                  item: @config['windows_auth_vault']['item'],
+                  field: @config["windows_auth_vault"]["sshd_password_field"]
+                )
+              end
+            end
+
+            win_admin_password ||= MU.generateWindowsPassword
+            ec2config_password ||= MU.generateWindowsPassword
+            sshd_password ||= MU.generateWindowsPassword
+
+            # We're creating the vault here so when we run
+            # MU::Cloud::Server.initialSSHTasks and we need to set the Windows
+            # Admin password we can grab it from said vault.
+            creds = {
+              "username" => @config['windows_admin_username'],
+              "password" => win_admin_password,
+              "ec2config_username" => "ec2config",
+              "ec2config_password" => ec2config_password,
+              "sshd_username" => "sshd_service",
+              "sshd_password" => sshd_password
+            }
+            @groomer.saveSecret(vault: @mu_name, item: "windows_credentials", data: creds, permissions: "name:#{@mu_name}")
+          end
+        end
+
+        def haveElasticIP?
+          if !cloud_desc.public_ip_address.nil?
+            begin
+              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(public_ips: [cloud_desc.public_ip_address])
+              if resp.addresses.size > 0 and resp.addresses.first.instance_id == @cloud_id
+                return true
+              end
+            rescue Aws::EC2::Errors::InvalidAddressNotFound
+              # XXX this is ok to ignore, it means the public IP isn't Elastic
+            end
+          end
+
+          false
+        end
+
+        def configureNetworking
+          if !@config['static_ip'].nil?
+            if !@config['static_ip']['ip'].nil?
+              MU::Cloud::AWS::Server.associateElasticIp(@cloud_id, classic: @vpc.nil?, ip: @config['static_ip']['ip'])
+            elsif !haveElasticIP?
+              MU::Cloud::AWS::Server.associateElasticIp(@cloud_id, classic: @vpc.nil?)
+            end
+          end
+
+          if !@vpc.nil? and @config.has_key?("vpc")
+            subnet = @vpc.getSubnet(cloud_id: cloud_desc.subnet_id)
+
+            _nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
+            if subnet.private? and !nat_ssh_host and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
+              raise MuError, "#{@mu_name} is in a private subnet (#{subnet}), but has no bastion host configured, and I have no other route to it"
+            end
+
+            # If we've asked for additional subnets (and this @config is not a
+            # member of a Server Pool, which has different semantics), create
+            # extra interfaces to accomodate.
+            if !@config['vpc']['subnets'].nil? and @config['basis'].nil?
+              device_index = 1
+              mySubnets.each { |s|
+                next if s.cloud_id == cloud_desc.subnet_id
+
+                if cloud_desc.placement.availability_zone != s.az
+                  MU.log "Cannot create interface in subnet #{s.to_s} for #{@mu_name} due to AZ mismatch", MU::WARN
+                  next
+                end
+                MU.log "Adding network interface on subnet #{s.cloud_id} for #{@mu_name}"
+                iface = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_network_interface(subnet_id: s.cloud_id).network_interface
+                MU::Cloud::AWS.createStandardTags(
+                  iface.network_interface_id,
+                  region: @config['region'],
+                  credentials: @config['credentials'],
+                  optional: @config['optional_tags'],
+                  nametag: @mu_name+"-ETH"+device_index.to_s,
+                  othertags: @config['tags']
+                )
+
+                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_network_interface(
+                  network_interface_id: iface.network_interface_id,
+                  instance_id: cloud_desc.instance_id,
+                  device_index: device_index
+                )
+                device_index = device_index + 1
+              }
+              cloud_desc(use_cache: false)
+            end
+          end
+
+          [:private_dns_name, :public_dns_name, :private_ip_address, :public_ip_address].each { |field|
+            @config[field.to_s] = cloud_desc.send(field)
+          }
+
+          if !@config['add_private_ips'].nil?
+            cloud_desc.network_interfaces.each { |int|
+              if int.private_ip_address == cloud_desc.private_ip_address and int.private_ip_addresses.size < (@config['add_private_ips'] + 1)
+                MU.log "Adding #{@config['add_private_ips']} extra private IP addresses to #{cloud_desc.instance_id}"
+                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).assign_private_ip_addresses(
+                  network_interface_id: int.network_interface_id,
+                  secondary_private_ip_address_count: @config['add_private_ips'],
+                  allow_reassignment: false
+                )
+              end
+            }
+          end
+        end
+
+        def tagVolumes
+          volumes = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(filters: [name: "attachment.instance-id", values: [@cloud_id]])
+          volumes.each { |vol|
+            vol.volumes.each { |volume|
+              volume.attachments.each { |attachment|
+                MU::Cloud::AWS.createStandardTags(
+                  attachment.volume_id,
+                  region: @config['region'],
+                  credentials: @config['credentials'],
+                  optional: @config['optional_tags'],
+                  nametag: ["/dev/sda", "/dev/sda1"].include?(attachment.device) ? "ROOT-"+@mu_name : @mu_name+"-"+attachment.device.upcase,
+                  othertags: @config['tags']
+                )
+  
+              }
+            }
+          }
+        end
+
+        # If we came up via AutoScale, the Alarm module won't have had our
+        # instance ID to associate us with itself. So invoke that here.
+        # XXX might be possible to do this with regular alarm resources and
+        # dependencies now
+        def setAlarms
+          if !@config['basis'].nil? and @config["alarms"] and !@config["alarms"].empty?
+            @config["alarms"].each { |alarm|
+              alarm_obj = MU::MommaCat.findStray(
+                "AWS",
+                "alarms",
+                region: @config["region"],
+                deploy_id: @deploy.deploy_id,
+                name: alarm['name']
+              ).first
+              alarm["dimensions"] = [{:name => "InstanceId", :value => @cloud_id}]
+
+              if alarm["enable_notifications"]
+                topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"], credentials: @config['credentials'])
+                MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"], credentials: @config["credentials"])
+                alarm["alarm_actions"] = [topic_arn]
+                alarm["ok_actions"]  = [topic_arn]
+              end
+
+              alarm_name = alarm_obj ? alarm_obj.cloud_id : "#{@mu_name}-#{alarm['name']}".upcase
+
+              MU::Cloud::AWS::Alarm.setAlarm(
+                name: alarm_name,
+                ok_actions: alarm["ok_actions"],
+                alarm_actions: alarm["alarm_actions"],
+                insufficient_data_actions: alarm["no_data_actions"],
+                metric_name: alarm["metric_name"],
+                namespace: alarm["namespace"],
+                statistic: alarm["statistic"],
+                dimensions: alarm["dimensions"],
+                period: alarm["period"],
+                unit: alarm["unit"],
+                evaluation_periods: alarm["evaluation_periods"],
+                threshold: alarm["threshold"],
+                comparison_operator: alarm["comparison_operator"],
+                region: @config["region"],
+                credentials: @config['credentials']
+              )
+            }
+          end
+        end
+
+        # We have issues sometimes where our dns_records are pointing at the wrong node name and IP address.
+
+        def getIAMProfile
+          arn = if @config['generate_iam_role']
+            role = @deploy.findLitterMate(name: @config['name'], type: "roles")
+            s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
+              'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(@credentials)+'/'+file
+            }
+            MU.log "Adding S3 read permissions to #{@mu_name}'s IAM profile", MU::NOTICE, details: s3_objs
+            role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
+  
+            @config['iam_role'] = role.mu_name
+            role.cloudobj.createInstanceProfile
+  
+          elsif @config['iam_role'].nil?
+            raise MuError, "#{@mu_name} has generate_iam_role set to false, but no iam_role assigned."
+          end
+
+          if !@config["iam_role"].nil?
+            if arn
+              return {arn: arn}
+            else
+              return {name: @config["iam_role"]}
+            end
+          end
+
+          nil
+        end
+
+        def setDeleteOntermination(device, delete_on_termination = false)
+          mappings = MU.structToHash(cloud_desc.block_device_mappings)
+          mappings.each { |vol|
+            if vol[:ebs]
+              vol[:ebs].delete(:attach_time)
+              vol[:ebs].delete(:status)
+            end
+            if vol[:device_name] == device
+              if vol[:ebs][:delete_on_termination] != delete_on_termination
+                vol[:ebs][:delete_on_termination] = delete_on_termination
+                MU.log "Setting delete_on_termination flag to #{delete_on_termination.to_s} on #{@mu_name}'s #{dev}"
+                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
+                  instance_id: @cloud_id,
+                  block_device_mappings: mappings
+                )
+              end
+              return true
+            end
+          }
+
+          false
+        end
+
+        def createImage
+          img_cfg = @config['create_image']
+          # Scrub things that don't belong on an AMI
+          session = windows? ? getWinRMSession : getSSHSession
+          sudo = purgecmd = ""
+          sudo = "sudo" if @config['ssh_user'] != "root"
+          if windows?
+            purgecmd = "rm -rf /cygdrive/c/mu_installed_chef"
+          else
+            purgecmd = "rm -rf /opt/mu_installed_chef"
+          end
+          if img_cfg['image_then_destroy']
+            if windows?
+              purgecmd = "rm -rf /cygdrive/c/chef/ /home/#{@config['windows_admin_username']}/.ssh/authorized_keys /home/Administrator/.ssh/authorized_keys /cygdrive/c/mu-installer-ran-updates /cygdrive/c/mu_installed_chef"
+              # session.exec!("powershell -Command \"& {(Get-WmiObject -Class Win32_Product -Filter \"Name='UniversalForwarder'\").Uninstall()}\"")
+            else
+              purgecmd = "#{sudo} rm -rf /var/lib/cloud/instances/i-* /root/.ssh/authorized_keys /etc/ssh/ssh_host_*key* /etc/chef /etc/opscode/* /.mu-installer-ran-updates /var/chef /opt/mu_installed_chef /opt/chef ; #{sudo} sed -i 's/^HOSTNAME=.*//' /etc/sysconfig/network"
+            end
+          end
+          if windows?
+            session.run(purgecmd)
+          else
+            session.exec!(purgecmd)
+          end
+          session.close
+          ami_ids = MU::Cloud::AWS::Server.createImage(
+              name: @mu_name,
+              instance_id: @cloud_id,
+              storage: @config['storage'],
+              exclude_storage: img_cfg['image_exclude_storage'],
+              copy_to_regions: img_cfg['copy_to_regions'],
+              make_public: img_cfg['public'],
+              region: @config['region'],
+              tags: @config['tags'],
+              credentials: @config['credentials']
+          )
+          @deploy.notify("images", @config['name'], ami_ids)
+          @config['image_created'] = true
+          if img_cfg['image_then_destroy']
+            MU::Cloud::AWS::Server.waitForAMI(ami_ids[@config['region']], region: @config['region'], credentials: @config['credentials'])
+            MU.log "AMI #{ami_ids[@config['region']]} ready, removing source node #{@mu_name}"
+            MU::Cloud::AWS::Server.terminateInstance(id: @cloud_id, region: @config['region'], deploy_id: @deploy.deploy_id, mu_name: @mu_name, credentials: @config['credentials'])
+            destroy
           end
         end
-        private_class_method :delete_volume
 
       end #class
     end #class