cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

data/modules/mu/clouds/aws/search_domain.rb

@@ -42,7 +42,8 @@ module MU
           params = genParams
 
           MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params
-          resp = MU::Cloud::AWS.elasticsearch(@config['region']).create_elasticsearch_domain(params).domain_status
+          pp params
+          resp = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).create_elasticsearch_domain(params).domain_status
 
           tagDomain
 
@@ -58,7 +59,7 @@ module MU
             waitWhileProcessing # wait until the create finishes, if still going
 
             MU.log "Updating ElasticSearch domain #{@config['domain_name']}", MU::NOTICE, details: params
-            MU::Cloud::AWS.elasticsearch(@config['region']).update_elasticsearch_domain_config(params)
+            MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).update_elasticsearch_domain_config(params)
           end
 
           waitWhileProcessing # don't return until creation/updating is complete
@@ -69,11 +70,11 @@ module MU
         # our druthers.
         def cloud_desc
           if @config['domain_name']
-            MU::Cloud::AWS.elasticsearch(@config['region']).describe_elasticsearch_domain(
+            MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).describe_elasticsearch_domain(
               domain_name: @config['domain_name']
             ).domain_status
-          elsif @deploydata['domain_name']
-            MU::Cloud::AWS.elasticsearch(@config['region']).describe_elasticsearch_domain(
+          elsif @deploydata and @deploydata['domain_name']
+            MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).describe_elasticsearch_domain(
               domain_name: @deploydata['domain_name']
             ).domain_status
           else
@@ -91,7 +92,7 @@ module MU
         # @return [Hash]
         def notify
           deploy_struct = MU.structToHash(cloud_desc)
-          tags = MU::Cloud::AWS.elasticsearch(@config['region']).list_tags(arn: deploy_struct[:arn]).tag_list
+          tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).list_tags(arn: deploy_struct[:arn]).tag_list
           deploy_struct['tags'] = tags.map { |t| { t.key => t.value } }
           if deploy_struct['endpoint']
             deploy_struct['kibana'] = deploy_struct['endpoint']+"/_plugin/kibana/"
@@ -100,23 +101,30 @@ module MU
           deploy_struct
         end
 
+        # Does this resource type exist as a global (cloud-wide) artifact, or
+        # is it localized to a region/zone?
+        # @return [Boolean]
+        def self.isGlobal?
+          false
+        end
+
         # Remove all search_domains associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
-          list = MU::Cloud::AWS.elasticsearch(region).list_domain_names
+        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          list = MU::Cloud::AWS.elasticsearch(region: region).list_domain_names
           if list and list.domain_names and list.domain_names.size > 0
-            descs = MU::Cloud::AWS.elasticsearch(region).describe_elasticsearch_domains(domain_names: list.domain_names.map { |d| d.domain_name } )
+            descs = MU::Cloud::AWS.elasticsearch(region: region).describe_elasticsearch_domains(domain_names: list.domain_names.map { |d| d.domain_name } )
 
             descs.domain_status_list.each { |domain|
-              tags = MU::Cloud::AWS.elasticsearch(region).list_tags(arn: domain.arn)
+              tags = MU::Cloud::AWS.elasticsearch(region: region).list_tags(arn: domain.arn)
               tags.tag_list.each { |tag|
                 if tag.key == "MU-ID" and tag.value == MU.deploy_id
                   MU.log "Deleting ElasticSearch Domain #{domain.domain_name}"
                   if !noop
-                    MU::Cloud::AWS.elasticsearch(region).delete_elasticsearch_domain(domain_name: domain.domain_name)
+                    MU::Cloud::AWS.elasticsearch(region: region).delete_elasticsearch_domain(domain_name: domain.domain_name)
                   end
                   break
                 end
@@ -142,14 +150,14 @@ module MU
         # @param region [String]: The cloud provider region.
         # @param flags [Hash]: Optional flags
         # @return [OpenStruct]: The cloud provider's complete descriptions of matching search_domain.
-        def self.find(cloud_id: nil, region: MU.curRegion, flags: {})
+        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
           if cloud_id
             # Annoyingly, we might expect one of several possible artifacts,
             # since AWS couldn't decide what the real identifier of these
             # things should be
-            list = MU::Cloud::AWS.elasticsearch(region).list_domain_names
+            list = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).list_domain_names
             if list and list.domain_names and list.domain_names.size > 0
-              descs = MU::Cloud::AWS.elasticsearch(region).describe_elasticsearch_domains(domain_names: list.domain_names.map { |d| d.domain_name } )
+              descs = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).describe_elasticsearch_domains(domain_names: list.domain_names.map { |d| d.domain_name } )
               descs.domain_status_list.each { |domain|
                 return domain if domain.arn == cloud_id
                 return domain if domain.domain_name == cloud_id
@@ -180,6 +188,10 @@ module MU
           end
 
           schema = {
+            "name" => {
+              "type" => "string",
+              "pattern" => '^[a-z][a-z0-9\-]+$'
+            },
             "elasticsearch_version" => {
               "type" => "string",
               "default" => versions.first,
@@ -269,12 +281,12 @@ module MU
         # @return [Boolean]: True if validation succeeded, False otherwise
         def self.validateConfig(dom, configurator)
           ok = true
-          versions = MU::Cloud::AWS.elasticsearch(dom['region']).list_elasticsearch_versions.elasticsearch_versions
+          versions = MU::Cloud::AWS.elasticsearch(region: dom['region']).list_elasticsearch_versions.elasticsearch_versions
           if !versions.include?(dom["elasticsearch_version"])
             MU.log "Invalid ElasticSearch version '#{dom["elasticsearch_version"]}' in SearchDomain '#{dom['name']}'", MU::ERR, details: versions
             ok = false
           else
-            resp = MU::Cloud::AWS.elasticsearch(dom['region']).list_elasticsearch_instance_types(
+            resp = MU::Cloud::AWS.elasticsearch(region: dom['region']).list_elasticsearch_instance_types(
               elasticsearch_version: dom["elasticsearch_version"]
             )
           
@@ -330,7 +342,7 @@ module MU
             if configurator.haveLitterMate?(dom['slow_logs'], "log")
               dom['dependencies'] << { "name" => dom['slow_logs'], "type" => "log" }
             else
-              log_group = MU::Cloud::AWS::Log.find(cloud_id: dom['slow_logs'], region: dom['region'])
+              log_group = MU::Cloud::AWS::Log.find(cloud_id: dom['slow_logs'], region: dom['region']).values.first
               if !log_group
                 MU.log "Specified slow_logs CloudWatch log group '#{dom['slow_logs']}' in SearchDomain '#{dom['name']}' doesn't appear to exist", MU::ERR
                 ok = false
@@ -340,7 +352,10 @@ module MU
             end
           else
             dom['slow_logs'] = dom['name']+"-slowlog"
-            log_group = { "name" => dom['slow_logs'] }
+            log_group = {
+              "name" => dom['slow_logs'],
+              "credentials" => dom['credentials']
+            }
             ok = false if !configurator.insertKitten(log_group, "logs")
             dom['dependencies'] << { "name" => dom['slow_logs'], "type" => "log" }
           end
@@ -353,7 +368,7 @@ module MU
 
           if dom['cognito']
             begin
-              MU::Cloud::AWS.cognito_ident(dom['region']).describe_identity_pool(
+              MU::Cloud::AWS.cognito_ident(region: dom['region']).describe_identity_pool(
                 identity_pool_id: dom['cognito']['identity_pool_id']
               )
             rescue ::Aws::CognitoIdentity::Errors::ValidationException, Aws::CognitoIdentity::Errors::ResourceNotFoundException => e
@@ -361,7 +376,7 @@ module MU
               ok = false
             end
             begin
-              MU::Cloud::AWS.cognito_user(dom['region']).describe_user_pool(
+              MU::Cloud::AWS.cognito_user(region: dom['region']).describe_user_pool(
                 user_pool_id: dom['cognito']['user_pool_id']
               )
             rescue ::Aws::CognitoIdentityProvider::Errors::InvalidParameterException, Aws::CognitoIdentityProvider::Errors::ResourceNotFoundException => e
@@ -373,10 +388,10 @@ module MU
               rolename = dom['cognito']['role_arn'].sub(/.*?:role\/([a-z0-9-]+)$/, '\1')
               begin
                 if !dom['cognito']['role_arn'].match(/^arn:/)
-                  role = MU::Cloud::AWS.iam(dom['region']).get_role(role_name: rolename)
+                  role = MU::Cloud::AWS.iam.get_role(role_name: rolename)
                   dom['cognito']['role_arn'] = role.role.arn
                 end
-                pols = MU::Cloud::AWS.iam(dom['region']).list_attached_role_policies(role_name: rolename).attached_policies
+                pols = MU::Cloud::AWS.iam.list_attached_role_policies(role_name: rolename).attached_policies
                 found = false
                 pols.each { |policy|
                   found = true if policy.policy_name == "AmazonESCognitoAccess"
@@ -391,6 +406,7 @@ module MU
             else
               roledesc = {
                 "name" => dom['name']+"cognitorole",
+                "credentials" => dom['credentials'],
                 "can_assume" => [
                   {
                     "entity_id" => "es.amazonaws.com",
@@ -471,7 +487,7 @@ module MU
               arn = @config['slow_logs']
             else
               log_group = @deploy.findLitterMate(type: "log", name: @config['slow_logs'])
-              log_group = MU::Cloud::AWS::Log.find(cloud_id: log_group.mu_name, region: log_group.cloudobj.config['region'])
+              log_group = MU::Cloud::AWS::Log.find(cloud_id: log_group.mu_name, region: log_group.cloudobj.config['region']).values.first
               if log_group.nil? or log_group.arn.nil?
                 raise MuError, "Failed to retrieve ARN of sibling LogGroup '#{@config['slow_logs']}'"
               end
@@ -619,7 +635,7 @@ module MU
             raise MU::MuError, "Can't tag ElasticSearch domain, cloud descriptor came back without an ARN"
           end
 
-          MU::Cloud::AWS.elasticsearch(@config['region']).add_tags(
+          MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).add_tags(
             arn: domain.arn,
             tag_list: tags
           )

data/modules/mu/clouds/aws/server.rb

@@ -210,15 +210,15 @@ module MU
         # @param tag_value [String]: The value of the tag to attach.
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.tagVolumes(instance_id, device: nil, tag_name: "MU-ID", tag_value: MU.deploy_id, region: MU.curRegion)
-          MU::Cloud::AWS.ec2(region).describe_volumes(filters: [name: "attachment.instance-id", values: [instance_id]]).each { |vol|
+        def self.tagVolumes(instance_id, device: nil, tag_name: "MU-ID", tag_value: MU.deploy_id, region: MU.curRegion, credentials: nil)
+          MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_volumes(filters: [name: "attachment.instance-id", values: [instance_id]]).each { |vol|
             vol.volumes.each { |volume|
               volume.attachments.each { |attachment|
                 vol_parent = attachment.instance_id
                 vol_id = attachment.volume_id
                 vol_dev = attachment.device
                 if vol_parent == instance_id and (vol_dev == device or device.nil?)
-                  MU::MommaCat.createTag(vol_id, tag_name, tag_value, region: region)
+                  MU::MommaCat.createTag(vol_id, tag_name, tag_value, region: region, credentials: credentials)
                   break
                 end
               }
@@ -246,8 +246,8 @@ module MU
               end
               MU::MommaCat.unlock(instance.instance_id+"-create")
             else
-              MU::MommaCat.createStandardTags(instance.instance_id, region: @config['region'])
-              MU::MommaCat.createTag(instance.instance_id, "Name", @mu_name, region: @config['region'])
+              MU::MommaCat.createStandardTags(instance.instance_id, region: @config['region'], credentials: @config['credentials'])
+              MU::MommaCat.createTag(instance.instance_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
             end
             done = true
           rescue Exception => e
@@ -258,7 +258,7 @@ module MU
                 parent_thread_id = Thread.current.object_id
                 Thread.new {
                   MU.dupGlobals(parent_thread_id)
-                  MU::Cloud::AWS::Server.cleanup(noop: false, ignoremaster: false, skipsnapshots: true)
+                  MU::Cloud::AWS::Server.cleanup(noop: false, ignoremaster: false, region: @config['region'], credentials: @config['credentials'], flags: { "skipsnapshots" => true } )
                 }
               end
             end
@@ -286,7 +286,6 @@ module MU
 
           arn = nil
           if @config['generate_iam_role']
-#            @config['iam_role'], @cfm_role_name, @cfm_prof_name, arn = MU::Cloud::AWS::Server.createIAMProfile(@mu_name, base_profile: @config['iam_role'], extra_policies: @config['iam_policies'])
             role = @deploy.findLitterMate(name: @config['name'], type: "roles")
             s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
               'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU.adminBucketName+'/'+file
@@ -349,10 +348,10 @@ module MU
             instance_descriptor[:user_data] = Base64.encode64(@userdata)
           end
 
-          MU::Cloud::AWS::Server.waitForAMI(@config["ami_id"], region: @config['region'])
+          MU::Cloud::AWS::Server.waitForAMI(@config["ami_id"], region: @config['region'], credentials: @config['credentials'])
 
           # Figure out which devices are embedded in the AMI already.
-          image = MU::Cloud::AWS.ec2(@config['region']).describe_images(image_ids: [@config["ami_id"]]).images.first
+          image = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_images(image_ids: [@config["ami_id"]]).images.first
           ext_disks = {}
           if !image.block_device_mappings.nil?
             image.block_device_mappings.each { |disk|
@@ -391,7 +390,7 @@ module MU
 
           retries = 0
           begin
-            response = MU::Cloud::AWS.ec2(@config['region']).run_instances(instance_descriptor)
+            response = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).run_instances(instance_descriptor)
           rescue Aws::EC2::Errors::InvalidGroupNotFound, Aws::EC2::Errors::InvalidSubnetIDNotFound, Aws::EC2::Errors::InvalidParameterValue => e
             if retries < 10
               if retries > 7
@@ -419,40 +418,40 @@ module MU
           if hard
             groupname = nil
             if !@config['basis'].nil?
-              resp = MU::Cloud::AWS.autoscale(@config['region']).describe_auto_scaling_instances(
+              resp = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_instances(
                 instance_ids: [@cloud_id]
               )
               groupname = resp.auto_scaling_instances.first.auto_scaling_group_name
               MU.log "Pausing Autoscale processes in #{groupname}", MU::NOTICE
-              MU::Cloud::AWS.autoscale(@config['region']).suspend_processes(
+              MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).suspend_processes(
                 auto_scaling_group_name: groupname
               )
             end
             begin
               MU.log "Stopping #{@mu_name} (#{@cloud_id})", MU::NOTICE
-              MU::Cloud::AWS.ec2(@config['region']).stop_instances(
+              MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).stop_instances(
                 instance_ids: [@cloud_id]
               )
-              MU::Cloud::AWS.ec2(@config['region']).wait_until(:instance_stopped, instance_ids: [@cloud_id]) do |waiter|
+              MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).wait_until(:instance_stopped, instance_ids: [@cloud_id]) do |waiter|
                 waiter.before_attempt do |attempts|
                   MU.log "Waiting for #{@mu_name} to stop for hard reboot"
                 end
               end
               MU.log "Starting #{@mu_name} (#{@cloud_id})"
-              MU::Cloud::AWS.ec2(@config['region']).start_instances(
+              MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).start_instances(
                 instance_ids: [@cloud_id]
               )
             ensure
               if !groupname.nil?
                 MU.log "Resuming Autoscale processes in #{groupname}", MU::NOTICE
-                MU::Cloud::AWS.autoscale(@config['region']).resume_processes(
+                MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).resume_processes(
                   auto_scaling_group_name: groupname
                 )
               end
             end
           else
             MU.log "Rebooting #{@mu_name} (#{@cloud_id})"
-            MU::Cloud::AWS.ec2(@config['region']).reboot_instances(
+            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).reboot_instances(
               instance_ids: [@cloud_id]
             )
           end
@@ -468,7 +467,7 @@ module MU
           return nil if @config.nil? or @deploy.nil?
 
           nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
-          if !@config["vpc"].nil? and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'])
+          if !@config["vpc"].nil? and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
             if !@nat.nil?
               if @nat.is_a?(Struct) && @nat.nat_gateway_id && @nat.nat_gateway_id.start_with?("nat-")
                 raise MuError, "Configured to use NAT Gateway, but I have no route to instance. Either use Bastion, or configure VPC peering"
@@ -516,18 +515,18 @@ module MU
           return false if !MU::MommaCat.lock(instance.instance_id+"-orchestrate", true)
           return false if !MU::MommaCat.lock(instance.instance_id+"-groom", true)
 
-          MU::MommaCat.createStandardTags(instance.instance_id, region: @config['region'])
-          MU::MommaCat.createTag(instance.instance_id, "Name", node, region: @config['region'])
+          MU::MommaCat.createStandardTags(instance.instance_id, region: @config['region'], credentials: @config['credentials'])
+          MU::MommaCat.createTag(instance.instance_id, "Name", node, region: @config['region'], credentials: @config['credentials'])
 
           if @config['optional_tags']
             MU::MommaCat.listOptionalTags.each { |key, value|
-              MU::MommaCat.createTag(instance.instance_id, key, value, region: @config['region'])
+              MU::MommaCat.createTag(instance.instance_id, key, value, region: @config['region'], credentials: @config['credentials'])
             }
           end
 
           if !@config['tags'].nil?
             @config['tags'].each { |tag|
-              MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: @config['region'])
+              MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
             }
           end
           MU.log "Tagged #{node} (#{instance.instance_id}) with MU-ID=#{MU.deploy_id}", MU::DEBUG
@@ -552,7 +551,7 @@ module MU
               end
               sleep 40
               # Get a fresh AWS descriptor
-              instance = MU::Cloud::Server.find(cloud_id: @cloud_id, region: @config['region']).values.first
+              instance = MU::Cloud::Server.find(cloud_id: @cloud_id, region: @config['region'], credentials: @config['credentials']).values.first
               if instance and instance.state.name == "terminated"
                 raise MuError, "EC2 instance #{node} (#{@cloud_id}) terminating during bootstrap!"
               end
@@ -572,6 +571,8 @@ module MU
 
           # If we came up via AutoScale, the Alarm module won't have had our
           # instance ID to associate us with itself. So invoke that here.
+          # XXX might be possible to do this with regular alarm resources and
+          # dependencies now
           if !@config['basis'].nil? and @config["alarms"] and !@config["alarms"].empty?
             @config["alarms"].each { |alarm|
               alarm_obj = MU::MommaCat.findStray(
@@ -584,8 +585,8 @@ module MU
               alarm["dimensions"] = [{:name => "InstanceId", :value => @cloud_id}]
 
               if alarm["enable_notifications"]
-                topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"])
-                MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"])
+                topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"], credentials: @config['credentials'])
+                MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"], credentials: @config["credentials"])
                 alarm["alarm_actions"] = [topic_arn]
                 alarm["ok_actions"]  = [topic_arn]
               end
@@ -606,7 +607,8 @@ module MU
                 evaluation_periods: alarm["evaluation_periods"],
                 threshold: alarm["threshold"],
                 comparison_operator: alarm["comparison_operator"],
-                region: @config["region"]
+                region: @config["region"],
+                credentials: @config['credentials']
               )
             }
           end
@@ -635,7 +637,7 @@ module MU
 
           if !@config['src_dst_check'] and !@config["vpc"].nil?
             MU.log "Disabling source_dest_check #{node} (making it NAT-worthy)"
-            MU::Cloud::AWS.ec2(@config['region']).modify_instance_attribute(
+            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
                 instance_id: @cloud_id,
                 source_dest_check: {:value => false}
             )
@@ -643,7 +645,7 @@ module MU
 
           # Set console termination protection. Autoscale nodes won't set this
           # by default.
-          MU::Cloud::AWS.ec2(@config['region']).modify_instance_attribute(
+          MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
               instance_id: @cloud_id,
               disable_api_termination: {:value => true}
           )
@@ -651,7 +653,7 @@ module MU
           has_elastic_ip = false
           if !instance.public_ip_address.nil?
             begin
-              resp = MU::Cloud::AWS.ec2((@config['region'])).describe_addresses(public_ips: [instance.public_ip_address])
+              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(public_ips: [instance.public_ip_address])
               if resp.addresses.size > 0 and resp.addresses.first.instance_id == @cloud_id
                 has_elastic_ip = true
               end
@@ -737,7 +739,7 @@ module MU
             end
 
             nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name = getSSHConfig
-            if subnet.private? and !nat_ssh_host and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'])
+            if subnet.private? and !nat_ssh_host and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
               raise MuError, "#{node} is in a private subnet (#{subnet}), but has no NAT host configured, and I have no other route to it"
             end
 
@@ -749,23 +751,23 @@ module MU
               @vpc.subnets { |subnet|
                 subnet_id = subnet.cloud_id
                 MU.log "Adding network interface on subnet #{subnet_id} for #{node}"
-                iface = MU::Cloud::AWS.ec2(@config['region']).create_network_interface(subnet_id: subnet_id).network_interface
-                MU::MommaCat.createStandardTags(iface.network_interface_id, region: @config['region'])
-                MU::MommaCat.createTag(iface.network_interface_id, "Name", node+"-ETH"+device_index.to_s, region: @config['region'])
+                iface = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_network_interface(subnet_id: subnet_id).network_interface
+                MU::MommaCat.createStandardTags(iface.network_interface_id, region: @config['region'], credentials: @config['credentials'])
+                MU::MommaCat.createTag(iface.network_interface_id, "Name", node+"-ETH"+device_index.to_s, region: @config['region'], credentials: @config['credentials'])
 
                 if @config['optional_tags']
                   MU::MommaCat.listOptionalTags.each { |key, value|
-                    MU::MommaCat.createTag(iface.network_interface_id, key, value, region: @config['region'])
+                    MU::MommaCat.createTag(iface.network_interface_id, key, value, region: @config['region'], credentials: @config['credentials'])
                   }
                 end
 
                 if !@config['tags'].nil?
                   @config['tags'].each { |tag|
-                    MU::MommaCat.createTag(iface.network_interface_id, tag['key'], tag['value'], region: @config['region'])
+                    MU::MommaCat.createTag(iface.network_interface_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
                   }
                 end
 
-                MU::Cloud::AWS.ec2(@config['region']).attach_network_interface(
+                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_network_interface(
                     network_interface_id: iface.network_interface_id,
                     instance_id: instance.instance_id,
                     device_index: device_index
@@ -805,29 +807,29 @@ module MU
 
           # Tag volumes with all our standard tags.
           # Maybe replace tagVolumes with this? There is one more place tagVolumes is called from
-          volumes = MU::Cloud::AWS.ec2(@config['region']).describe_volumes(filters: [name: "attachment.instance-id", values: [instance.instance_id]])
+          volumes = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(filters: [name: "attachment.instance-id", values: [instance.instance_id]])
           volumes.each { |vol|
             vol.volumes.each { |volume|
               volume.attachments.each { |attachment|
                 MU::MommaCat.listStandardTags.each_pair { |key, value|
-                  MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'])
+                  MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
 
                   if attachment.device == "/dev/sda" or attachment.device == "/dev/sda1"
-                    MU::MommaCat.createTag(attachment.volume_id, "Name", "ROOT-#{MU.deploy_id}-#{@config["name"].upcase}", region: @config['region'])
+                    MU::MommaCat.createTag(attachment.volume_id, "Name", "ROOT-#{MU.deploy_id}-#{@config["name"].upcase}", region: @config['region'], credentials: @config['credentials'])
                   else
-                    MU::MommaCat.createTag(attachment.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{attachment.device.upcase}", region: @config['region'])
+                    MU::MommaCat.createTag(attachment.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{attachment.device.upcase}", region: @config['region'], credentials: @config['credentials'])
                   end
                 }
 
                 if @config['optional_tags']
                   MU::MommaCat.listOptionalTags.each { |key, value|
-                    MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'])
+                    MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
                   }
                 end
 
                 if @config['tags']
                   @config['tags'].each { |tag|
-                    MU::MommaCat.createTag(attachment.volume_id, tag['key'], tag['value'], region: @config['region'])
+                    MU::MommaCat.createTag(attachment.volume_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
                   }
                 end
               }
@@ -842,7 +844,7 @@ module MU
             instance.network_interfaces.each { |int|
               if int.private_ip_address == instance.private_ip_address and int.private_ip_addresses.size < (@config['add_private_ips'] + 1)
                 MU.log "Adding #{@config['add_private_ips']} extra private IP addresses to #{instance.instance_id}"
-                MU::Cloud::AWS.ec2(@config['region']).assign_private_ip_addresses(
+                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).assign_private_ip_addresses(
                     network_interface_id: int.network_interface_id,
                     secondary_private_ip_address_count: @config['add_private_ips'],
                     allow_reassignment: false
@@ -853,31 +855,33 @@ module MU
           end
 
           begin
-            if windows?
-              # kick off certificate generation early; WinRM will need it
-              cert, key = @deploy.nodeSSLCerts(self)
-              if @config.has_key?("basis")
-                @deploy.nodeSSLCerts(self, true)
-              end
-              if !@groomer.haveBootstrapped?
-                session = getWinRMSession(50, 60, reboot_on_problems: true)
-                initialWinRMTasks(session)
-                begin
-                  session.close
-                rescue Exception
-                  # this is allowed to fail- we're probably rebooting anyway
+            if @config['groom'].nil? or @config['groom']
+              if windows?
+                # kick off certificate generation early; WinRM will need it
+                cert, key = @deploy.nodeSSLCerts(self)
+                if @config.has_key?("basis")
+                  @deploy.nodeSSLCerts(self, true)
                 end
-              else # for an existing Windows node: WinRM, then SSH if it fails
-                begin
-                  session = getWinRMSession(1, 60)
-                rescue Exception # yeah, yeah
-                  session = getSSHSession(1, 60)
-                  # XXX maybe loop at least once if this also fails?
+                if !@groomer.haveBootstrapped?
+                  session = getWinRMSession(50, 60, reboot_on_problems: true)
+                  initialWinRMTasks(session)
+                  begin
+                    session.close
+                  rescue Exception
+                    # this is allowed to fail- we're probably rebooting anyway
+                  end
+                else # for an existing Windows node: WinRM, then SSH if it fails
+                  begin
+                    session = getWinRMSession(1, 60)
+                  rescue Exception # yeah, yeah
+                    session = getSSHSession(1, 60)
+                    # XXX maybe loop at least once if this also fails?
+                  end
                 end
+              else
+                session = getSSHSession(40, 30)
+                initialSSHTasks(session)
               end
-            else
-              session = getSSHSession(40, 30)
-              initialSSHTasks(session)
             end
           rescue BootstrapTempFail
             sleep 45
@@ -922,14 +926,16 @@ module MU
           # we're done.
           if @groomer.haveBootstrapped?
             MU.log "Node #{node} has already been bootstrapped, skipping groomer setup.", MU::NOTICE
-            @groomer.saveDeployData
+            if @config['groom'].nil? or @config['groom']
+              @groomer.saveDeployData
+            end
             MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
             MU::MommaCat.unlock(instance.instance_id+"-groom")
             return true
           end
 
           begin
-            @groomer.bootstrap
+            @groomer.bootstrap if @config['groom'].nil? or @config['groom']
           rescue MU::Groomer::RunError
             MU::MommaCat.unlock(instance.instance_id+"-groom")
             MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
@@ -954,11 +960,11 @@ module MU
         # @param region [String]: The cloud provider region
         # @param tag_key [String]: A tag key to search.
         # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param ip [String]: An IP address associated with the instance
         # @param flags [Hash]: Optional flags
         # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching instances
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, ip: nil, flags: {})
+        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
 # XXX put that 'ip' value into opts
+          ip ||= flags['ip']
           instance = nil
           if !region.nil?
             regions = [region]
@@ -977,7 +983,7 @@ module MU
                 MU.log "Hunting for instance with cloud id '#{cloud_id}' in #{region}", MU::DEBUG
                 retries = 0
                 begin
-                  MU::Cloud::AWS.ec2(region).describe_instances(
+                  MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_instances(
                       instance_ids: [cloud_id],
                       filters: [
                           {name: "instance-state-name", values: ["running", "pending"]}
@@ -1018,7 +1024,7 @@ module MU
           if instance.nil? and !ip.nil?
             MU.log "Hunting for instance by IP '#{ip}'", MU::DEBUG
             ["ip-address", "private-ip-address"].each { |filter|
-              response = MU::Cloud::AWS.ec2(region).describe_instances(
+              response = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_instances(
                   filters: [
                       {name: filter, values: [ip]},
                       {name: "instance-state-name", values: ["running", "pending"]}
@@ -1035,7 +1041,7 @@ module MU
           # Fine, let's try it by tag.
           if !tag_value.nil?
             MU.log "Searching for instance by tag '#{tag_key}=#{tag_value}'", MU::DEBUG
-            MU::Cloud::AWS.ec2(region).describe_instances(
+            MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_instances(
                 filters: [
                     {name: "tag:#{tag_key}", values: [tag_value]},
                     {name: "instance-state-name", values: ["running", "pending"]}
@@ -1158,7 +1164,7 @@ module MU
 
           punchAdminNAT
 
-          MU::Cloud::AWS::Server.tagVolumes(@cloud_id)
+          MU::Cloud::AWS::Server.tagVolumes(@cloud_id, credentials: @config['credentials'])
 
           # If we have a loadbalancer configured, attach us to it
           if !@config['loadbalancers'].nil?
@@ -1182,10 +1188,14 @@ module MU
             # }
           # end
 
-          @groomer.saveDeployData
+          if @config['groom'].nil? or @config['groom']
+            @groomer.saveDeployData
+          end
 
           begin
-            @groomer.run(purpose: "Full Initial Run", max_retries: 15, reboot_first_fail: windows?)
+            if @config['groom'].nil? or @config['groom']
+              @groomer.run(purpose: "Full Initial Run", max_retries: 15, reboot_first_fail: windows?, timeout: @config['groomer_timeout'])
+            end
           rescue MU::Groomer::RunError => e
             MU.log "Proceeding after failed initial Groomer run, but #{node} may not behave as expected!", MU::WARN, details: e.message
           rescue Exception => e
@@ -1221,13 +1231,15 @@ module MU
                 copy_to_regions: img_cfg['copy_to_regions'],
                 make_public: img_cfg['public'],
                 region: @config['region'],
-                tags: @config['tags'])
+                tags: @config['tags'],
+                credentials: @config['credentials']
+            )
             @deploy.notify("images", @config['name'], {"image_id" => ami_id})
             @config['image_created'] = true
             if img_cfg['image_then_destroy']
-              MU::Cloud::AWS::Server.waitForAMI(ami_id, region: @config['region'])
+              MU::Cloud::AWS::Server.waitForAMI(ami_id, region: @config['region'], credentials: @config['credentials'])
               MU.log "AMI #{ami_id} ready, removing source node #{node}"
-              MU::Cloud::AWS::Server.terminateInstance(id: @cloud_id, region: @config['region'], deploy_id: @deploy.deploy_id, mu_name: @mu_name)
+              MU::Cloud::AWS::Server.terminateInstance(id: @cloud_id, region: @config['region'], deploy_id: @deploy.deploy_id, mu_name: @mu_name, credentials: @config['credentials'])
               destroy
             end
           end
@@ -1238,7 +1250,7 @@ module MU
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU.account_number+":instance/"+@cloud_id
+          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":instance/"+@cloud_id
         end
 
         def cloud_desc
@@ -1246,7 +1258,7 @@ module MU
           retries = 0
           if !@cloud_id.nil?
             begin
-              return MU::Cloud::AWS.ec2(@config['region']).describe_instances(instance_ids: [@cloud_id]).reservations.first.instances.first
+              return MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(instance_ids: [@cloud_id]).reservations.first.instances.first
             rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
               return nil
             rescue NoMethodError => e
@@ -1291,7 +1303,7 @@ module MU
           # Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
           # which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
           # The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
-          if MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region']) or @deploydata["public_ip_address"].nil?
+          if MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials']) or @deploydata["public_ip_address"].nil?
             @config['canonical_ip'] = instance.private_ip_address
             @deploydata["private_ip_address"] = instance.private_ip_address
             return instance.private_ip_address
@@ -1311,7 +1323,7 @@ module MU
         # @param copy_to_regions [Array<String>]: Copy the resulting AMI into the listed regions.
         # @param tags [Array<String>]: Extra/override tags to apply to the image.
         # @return [String]: The cloud provider identifier of the new machine image.
-        def self.createImage(name: nil, instance_id: nil, storage: {}, exclude_storage: false, make_public: false, region: MU.curRegion, copy_to_regions: [], tags: [])
+        def self.createImage(name: nil, instance_id: nil, storage: {}, exclude_storage: false, make_public: false, region: MU.curRegion, copy_to_regions: [], tags: [], credentials: nil)
           ami_descriptor = {
               :instance_id => instance_id,
               :name => name,
@@ -1344,18 +1356,18 @@ module MU
           MU.log "Creating AMI from #{name}", details: ami_descriptor
           resp = nil
           begin
-            resp = MU::Cloud::AWS.ec2(region).create_image(ami_descriptor)
+            resp = MU::Cloud::AWS.ec2(region: region).create_image(ami_descriptor)
           rescue Aws::EC2::Errors::InvalidAMINameDuplicate => e
             MU.log "AMI #{name} already exists, skipping", MU::WARN
             return nil
           end
           ami = resp.image_id
-          MU::MommaCat.createStandardTags(ami, region: region)
-          MU::MommaCat.createTag(ami, "Name", name, region: region)
+          MU::MommaCat.createStandardTags(ami, region: region, credentials: credentials)
+          MU::MommaCat.createTag(ami, "Name", name, region: region, credentials: credentials)
           MU.log "AMI of #{name} in region #{region}: #{ami}"
           if make_public
-            MU::Cloud::AWS::Server.waitForAMI(ami, region: region)
-            MU::Cloud::AWS.ec2(region).modify_image_attribute(
+            MU::Cloud::AWS::Server.waitForAMI(ami, region: region, credentials: credentials)
+            MU::Cloud::AWS.ec2(region: region).modify_image_attribute(
                 image_id: ami,
                 launch_permission: {add: [{group: "all"}]},
                 attribute: "launchPermission"
@@ -1364,12 +1376,12 @@ module MU
           copythreads = []
           if !copy_to_regions.nil? and copy_to_regions.size > 0
             parent_thread_id = Thread.current.object_id
-            MU::Cloud::AWS::Server.waitForAMI(ami, region: region) if !make_public
+            MU::Cloud::AWS::Server.waitForAMI(ami, region: region, credentials: credentials) if !make_public
             copy_to_regions.each { |r|
               next if r == region
               copythreads << Thread.new {
                 MU.dupGlobals(parent_thread_id)
-                copy = MU::Cloud::AWS.ec2(r).copy_image(
+                copy = MU::Cloud::AWS.ec2(region: r).copy_image(
                     source_region: region,
                     source_image_id: ami,
                     name: name,
@@ -1378,15 +1390,15 @@ module MU
                 MU.log "Initiated copy of #{ami} from #{region} to #{r}: #{copy.image_id}"
 
                 MU::MommaCat.createStandardTags(copy.image_id, region: r)
-                MU::MommaCat.createTag(copy.image_id, "Name", name, region: r)
+                MU::MommaCat.createTag(copy.image_id, "Name", name, region: r, credentials: credentials)
                 if !tags.nil?
                   tags.each { |tag|
-                    MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: r)
+                    MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: r, credentials: credentials)
                   }
                 end
-                MU::Cloud::AWS::Server.waitForAMI(copy.image_id, region: r)
+                MU::Cloud::AWS::Server.waitForAMI(copy.image_id, region: r, credentials: credentials)
                 if make_public
-                  MU::Cloud::AWS.ec2(r).modify_image_attribute(
+                  MU::Cloud::AWS.ec2(region: r).modify_image_attribute(
                       image_id: copy.image_id,
                       launch_permission: {add: [{group: "all"}]},
                       attribute: "launchPermission"
@@ -1408,12 +1420,12 @@ module MU
         # flagged as ready.
         # @param image_id [String]: The machine image to wait for.
         # @param region [String]: The cloud provider region
-        def self.waitForAMI(image_id, region: MU.curRegion)
+        def self.waitForAMI(image_id, region: MU.curRegion, credentials: nil)
           MU.log "Checking to see if AMI #{image_id} is available", MU::DEBUG
 
           retries = 0
           begin
-            images = MU::Cloud::AWS.ec2(region).describe_images(image_ids: [image_id]).images
+            images = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_images(image_ids: [image_id]).images
             if images.nil? or images.size == 0
               raise MuError, "No such AMI #{image_id} found"
             end
@@ -1497,7 +1509,7 @@ module MU
           retries = 0
           MU.log "Waiting for Windows instance password to be set by Amazon and flagged as available from the API. Note- if you're using a source AMI that already has its password set, this may fail. You'll want to set use_cloud_provider_windows_password to false if this is the case.", MU::NOTICE
           begin
-            MU::Cloud::AWS.ec2(@config['region']).wait_until(:password_data_available, instance_id: @cloud_id) do |waiter|
+            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).wait_until(:password_data_available, instance_id: @cloud_id) do |waiter|
               waiter.max_attempts = 60
               waiter.before_attempt do |attempts|
                 MU.log "Waiting for Windows password data to be available for node #{@mu_name}", MU::NOTICE if attempts % 5 == 0
@@ -1517,7 +1529,7 @@ module MU
             end
           end
 
-          resp = MU::Cloud::AWS.ec2(@config['region']).get_password_data(instance_id: @cloud_id)
+          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).get_password_data(instance_id: @cloud_id)
           encrypted_password = resp.password_data
 
           # Note: This is already implemented in the decrypt_windows_password API call
@@ -1544,9 +1556,9 @@ module MU
           filters << {name: "public-ip", values: [ip]} if ip != nil
 
           if filters.size > 0
-            resp = MU::Cloud::AWS.ec2(region).describe_addresses(filters: filters)
+            resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(filters: filters)
           else
-            resp = MU::Cloud::AWS.ec2(region).describe_addresses()
+            resp = MU::Cloud::AWS.ec2(region: region).describe_addresses()
           end
           resp.addresses.each { |address|
             return address if (address.network_interface_id.nil? || address.network_interface_id.empty?) && !@eips_used.include?(address.public_ip)
@@ -1559,10 +1571,10 @@ module MU
             end
           end
           if !classic
-            resp = MU::Cloud::AWS.ec2(region).allocate_address(domain: "vpc")
+            resp = MU::Cloud::AWS.ec2(region: region).allocate_address(domain: "vpc")
             new_ip = resp.public_ip
           else
-            new_ip = MU::Cloud::AWS.ec2(region).allocate_address().public_ip
+            new_ip = MU::Cloud::AWS.ec2(region: region).allocate_address().public_ip
           end
           filters = [{name: "public-ip", values: [new_ip]}]
           if resp.domain
@@ -1578,7 +1590,7 @@ module MU
           begin
             begin
               sleep 5
-              resp = MU::Cloud::AWS.ec2(region).describe_addresses(
+              resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(
                   filters: filters
               )
               addr = resp.addresses.first
@@ -1602,7 +1614,7 @@ module MU
             return true
           end
           az = nil
-          MU::Cloud::AWS.ec2(@config['region']).describe_instances(
+          MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(
             instance_ids: [@cloud_id]
           ).reservations.each { |resp|
             if !resp.nil? and !resp.instances.nil?
@@ -1618,14 +1630,14 @@ module MU
             end
           }
           MU.log "Creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
-          creation = MU::Cloud::AWS.ec2(@config['region']).create_volume(
+          creation = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_volume(
             availability_zone: az,
             size: size,
             volume_type: type
           )
           begin
             sleep 3
-            creation = MU::Cloud::AWS.ec2(@config['region']).describe_volumes(volume_ids: [creation.volume_id]).volumes.first
+            creation = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(volume_ids: [creation.volume_id]).volumes.first
             if !["creating", "available"].include?(creation.state)
               raise MuError, "Saw state '#{creation.state}' while creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
             end
@@ -1633,12 +1645,12 @@ module MU
 
           if @deploy
             MU::MommaCat.listStandardTags.each_pair { |key, value|
-              MU::MommaCat.createTag(creation.volume_id, key, value, region: @config['region'])
+              MU::MommaCat.createTag(creation.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
             }
-            MU::MommaCat.createTag(creation.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{dev.upcase}", region: @config['region'])
+            MU::MommaCat.createTag(creation.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{dev.upcase}", region: @config['region'], credentials: @config['credentials'])
           end
 
-          attachment = MU::Cloud::AWS.ec2(@config['region']).attach_volume(
+          attachment = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_volume(
             device: dev,
             instance_id: @cloud_id,
             volume_id: creation.volume_id
@@ -1646,7 +1658,7 @@ module MU
 
           begin
             sleep 3
-            attachment = MU::Cloud::AWS.ec2(@config['region']).describe_volumes(volume_ids: [attachment.volume_id]).volumes.first.attachments.first
+            attachment = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(volume_ids: [attachment.volume_id]).volumes.first.attachments.first
             if !["attaching", "attached"].include?(attachment.state)
               raise MuError, "Saw state '#{creation.state}' while creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
             end
@@ -1662,7 +1674,7 @@ module MU
             return true
           end
           begin
-            MU::Cloud::AWS.ec2(@config['region']).describe_instances(
+            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(
                 instance_ids: [@cloud_id]
             ).reservations.each { |resp|
               if !resp.nil? and !resp.instances.nil?
@@ -1694,7 +1706,7 @@ module MU
           @eip_semaphore.synchronize {
             if !ip.nil?
               filters = [{name: "public-ip", values: [ip]}]
-              resp = MU::Cloud::AWS.ec2(region).describe_addresses(filters: filters)
+              resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(filters: filters)
               if @eips_used.include?(ip)
                 is_free = false
                 resp.addresses.each { |address|
@@ -1726,12 +1738,12 @@ module MU
           attempts = 0
           begin
             if classic
-              resp = MU::Cloud::AWS.ec2(region).associate_address(
+              resp = MU::Cloud::AWS.ec2(region: region).associate_address(
                   instance_id: instance_id,
                   public_ip: elastic_ip.public_ip
               )
             else
-              resp = MU::Cloud::AWS.ec2(region).associate_address(
+              resp = MU::Cloud::AWS.ec2(region: region).associate_address(
                   instance_id: instance_id,
                   allocation_id: elastic_ip.allocation_id,
                   allow_reassociation: false
@@ -1747,7 +1759,7 @@ module MU
             raise MuError "#{e.message} associating #{elastic_ip.allocation_id} with #{instance_id}"
           rescue Aws::EC2::Errors::ResourceAlreadyAssociated => e
             # A previous association attempt may have succeeded, albeit slowly.
-            resp = MU::Cloud::AWS.ec2(region).describe_addresses(
+            resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(
                 allocation_ids: [elastic_ip.allocation_id]
             )
             first_addr = resp.addresses.first
@@ -1759,14 +1771,14 @@ module MU
             end
           end
 
-          instance = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
+          instance = MU::Cloud::AWS.ec2(region: region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
           waited = false
           if instance.public_ip_address != elastic_ip.public_ip
             waited = true
             begin
               sleep 10
               MU.log "Waiting for Elastic IP association of #{elastic_ip.public_ip} to #{instance_id} to take effect", MU::NOTICE
-              instance = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
+              instance = MU::Cloud::AWS.ec2(region: region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
             end while instance.public_ip_address != elastic_ip.public_ip
           end
 
@@ -1775,12 +1787,21 @@ module MU
           return elastic_ip.public_ip
         end
 
+        # Does this resource type exist as a global (cloud-wide) artifact, or
+        # is it localized to a region/zone?
+        # @return [Boolean]
+        def self.isGlobal?
+          false
+        end
+
         # Remove all instances associated with the currently loaded deployment. Also cleans up associated volumes, droppings in the MU master's /etc/hosts and ~/.ssh, and in whatever Groomer was used.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, skipsnapshots: false, onlycloud: false, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          onlycloud = flags["onlycloud"]
+          skipsnapshots = flags["skipsnapshots"]
           tagfilters = [
             {name: "tag:MU-ID", values: [MU.deploy_id]}
           ]
@@ -1794,7 +1815,7 @@ module MU
           # Build a list of instances we need to clean up. We guard against
           # accidental deletion here by requiring someone to have hand-terminated
           # these, by default.
-          resp = MU::Cloud::AWS.ec2(region).describe_instances(
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(
               filters: tagfilters
           )
 
@@ -1817,18 +1838,18 @@ module MU
             threads << Thread.new(instance) { |myinstance|
               MU.dupGlobals(parent_thread_id)
               Thread.abort_on_exception = true
-              MU::Cloud::AWS::Server.terminateInstance(id: myinstance.instance_id, noop: noop, onlycloud: onlycloud, region: region, deploy_id: MU.deploy_id)
+              MU::Cloud::AWS::Server.terminateInstance(id: myinstance.instance_id, noop: noop, onlycloud: onlycloud, region: region, deploy_id: MU.deploy_id, credentials: credentials)
             }
           }
 
-          resp = MU::Cloud::AWS.ec2(region).describe_volumes(
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_volumes(
               filters: tagfilters
           )
           resp.data.volumes.each { |volume|
             threads << Thread.new(volume) { |myvolume|
               MU.dupGlobals(parent_thread_id)
               Thread.abort_on_exception = true
-              MU::Cloud::AWS::Server.delete_volume(myvolume, noop, skipsnapshots)
+              MU::Cloud::AWS::Server.delete_volume(myvolume, noop, skipsnapshots, credentials: credentials)
             }
           }
 
@@ -1843,12 +1864,12 @@ module MU
         # @param id [String]: The cloud provider's identifier for the instance, to use if the full description is not available.
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.terminateInstance(instance: nil, noop: false, id: nil, onlycloud: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil)
+        def self.terminateInstance(instance: nil, noop: false, id: nil, onlycloud: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, credentials: nil)
           ips = Array.new
           if !instance
             if id
               begin
-                resp = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [id])
+                resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
               rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
                 MU.log "Instance #{id} no longer exists", MU::WARN
               end
@@ -1880,26 +1901,26 @@ module MU
           ).first
 
           begin
-            MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [id])
+            MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
           rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
             MU.log "Instance #{id} no longer exists", MU::DEBUG
           end
 
-          if !server_obj.nil? and MU::Cloud::AWS.hosted and !MU::Cloud::AWS.isGovCloud?
+          if !server_obj.nil? and MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud?
             # DNS cleanup is now done in MU::Cloud::DNSZone. Keeping this for now
             cleaned_dns = false
             mu_name = server_obj.mu_name
-            mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu").values.first
+            mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: credentials).values.first
             if !mu_zone.nil?
               zone_rrsets = []
-              rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: mu_zone.id)
+              rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: mu_zone.id)
               rrsets.resource_record_sets.each{ |record|
                 zone_rrsets << record
               }
 
             # AWS API returns a maximum of 100 results. DNS zones are likely to have more than 100 records, lets page and make sure we grab all records in a given zone
               while rrsets.next_record_name && rrsets.next_record_type
-                rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: mu_zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
+                rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: mu_zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
                 rrsets.resource_record_sets.each{ |record|
                   zone_rrsets << record
                 }
@@ -2002,14 +2023,14 @@ module MU
               MU.log "Terminating #{instance.instance_id} (#{name}) #{noop}"
               if !noop
                 begin
-                  MU::Cloud::AWS.ec2(region).modify_instance_attribute(
+                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_instance_attribute(
                       instance_id: instance.instance_id,
                       disable_api_termination: {value: false}
                   )
-                  MU::Cloud::AWS.ec2(region).terminate_instances(instance_ids: [instance.instance_id])
+                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).terminate_instances(instance_ids: [instance.instance_id])
                     # Small race window here with the state changing from under us
                 rescue Aws::EC2::Errors::IncorrectInstanceState => e
-                  resp = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [id])
+                  resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
                   if !resp.nil? and !resp.reservations.nil? and !resp.reservations.first.nil?
                     instance = resp.reservations.first.instances.first
                     if !instance.nil? and instance.state.name != "terminated" and instance.state.name != "terminating"
@@ -2026,7 +2047,7 @@ module MU
             end
             while instance.state.name != "terminated" and !noop
               sleep 30
-              instance_response = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [instance.instance_id])
+              instance_response = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [instance.instance_id])
               instance = instance_response.reservations.first.instances.first
             end
             MU.log "#{instance.instance_id} (#{name}) terminated" if !noop
@@ -2156,6 +2177,7 @@ module MU
           else
             role = {
               "name" => server["name"],
+              "credentials" => server["credentials"],
               "can_assume" => [
                 {
                   "entity_id" => "ec2.amazonaws.com",
@@ -2241,9 +2263,9 @@ module MU
         # @param id [String]: The cloud provider's identifier for the volume, to use if the full description is not available.
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.delete_volume(volume, noop, skipsnapshots, id: nil, region: MU.curRegion)
+        def self.delete_volume(volume, noop, skipsnapshots, id: nil, region: MU.curRegion, credentials: nil)
           if !volume.nil?
-            resp = MU::Cloud::AWS.ec2(region).describe_volumes(volume_ids: [volume.volume_id])
+            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_volumes(volume_ids: [volume.volume_id])
             volume = resp.data.volumes.first
           end
           name = ""
@@ -2260,15 +2282,25 @@ module MU
                 desc = "#{MU.deploy_id}-MUfinal"
               end
 
-              MU::Cloud::AWS.ec2(region).create_snapshot(
+              begin
+                MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_snapshot(
                   volume_id: volume.volume_id,
                   description: desc
-              )
+                )
+              rescue Aws::EC2::Errors::IncorrectState => e
+                if e.message.match(/'deleting'/)
+                  MU.log "Cannot snapshot volume '#{name}', is already being deleted", MU::WARN
+                end
+              end
             end
 
             retries = 0
             begin
-              MU::Cloud::AWS.ec2(region).delete_volume(volume_id: volume.volume_id)
+              MU::Cloud::AWS.ec2(region: region, credentials: credentials).delete_volume(volume_id: volume.volume_id)
+            rescue Aws::EC2::Errors::IncorrectState => e
+              MU.log "Volume #{volume.volume_id} (#{name}) in incorrect state (#{e.message}), will retry", MU::WARN
+              sleep 30
+              retry
             rescue Aws::EC2::Errors::InvalidVolumeNotFound
               MU.log "Volume #{volume.volume_id} (#{name}) disappeared before I could remove it!", MU::WARN
             rescue Aws::EC2::Errors::VolumeInUse