cloud-mu 3.3.0 → 3.5.1

data/modules/mu/providers/aws/container_cluster.rb

@@ -65,7 +65,7 @@ module MU
 
             on_retry = Proc.new { |e|
               # soul-crushing, yet effective
-              if e.message.match(/because (#{Regexp.quote(@config['region'])}[a-z]), the targeted availability zone, does not currently have sufficient capacity/)
+              if e.message.match(/because (#{Regexp.quote(@region)}[a-z]), the targeted availability zone, does not currently have sufficient capacity/)
                 bad_az = Regexp.last_match(1)
                 deletia = []
                 mySubnets.each { |subnet|
@@ -81,7 +81,7 @@ module MU
 
             MU.retrier([Aws::EKS::Errors::UnsupportedAvailabilityZoneException, Aws::EKS::Errors::InvalidParameterException], on_retry: on_retry, max: subnet_ids.size) {
               MU.log "Creating EKS cluster #{@mu_name}", details: params
-              MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).create_cluster(params)
+              MU::Cloud::AWS.eks(region: @region, credentials: @credentials).create_cluster(params)
             }
             @cloud_id = @mu_name
 
@@ -100,7 +100,7 @@ module MU
 
             MU.log "Creation of EKS cluster #{@mu_name} complete"
           else
-            MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).create_cluster(
+            MU::Cloud::AWS.ecs(region: @region, credentials: @credentials).create_cluster(
               cluster_name: @mu_name
             )
             @cloud_id = @mu_name
@@ -118,7 +118,7 @@ module MU
             # this account; EKS applications might want one, but will fail in
             # confusing ways if this hasn't been done.
             begin
-              MU::Cloud::AWS.iam(credentials: @config['credentials']).create_service_linked_role(
+              MU::Cloud::AWS.iam(credentials: @credentials).create_service_linked_role(
                 aws_service_name: "elasticloadbalancing.amazonaws.com"
               )
             rescue ::Aws::IAM::Errors::InvalidInput
@@ -170,7 +170,7 @@ module MU
             if tasks.size > 0
               tasks_failing = false
               MU.retrier(wait: 15, max: 10, loop_if: Proc.new { tasks_failing }){ |retries, _wait|
-                tasks_failing = !MU::Cloud::AWS::ContainerCluster.tasksRunning?(@mu_name, log: (retries > 0), region: @config['region'], credentials: @config['credentials'])
+                tasks_failing = !MU::Cloud::AWS::ContainerCluster.tasksRunning?(@mu_name, log: (retries > 0), region: @region, credentials: @credentials)
               }
 
               if tasks_failing
@@ -290,12 +290,12 @@ MU.log c.name, MU::NOTICE, details: t
           return nil if !@cloud_id
           @cloud_desc_cache = if @config['flavor'] == "EKS" or
              (@config['flavor'] == "Fargate" and !@config['containers'])
-            resp = MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).describe_cluster(
+            resp = MU::Cloud::AWS.eks(region: @region, credentials: @credentials).describe_cluster(
               name: @cloud_id
             )
             resp.cluster
           else
-            resp = MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).describe_clusters(
+            resp = MU::Cloud::AWS.ecs(region: @region, credentials: @credentials).describe_clusters(
               clusters: [@cloud_id]
             )
             resp.clusters.first
@@ -318,7 +318,7 @@ MU.log c.name, MU::NOTICE, details: t
         def notify
           deploy_struct = MU.structToHash(cloud_desc)
           deploy_struct['cloud_id'] = @mu_name
-          deploy_struct["region"] = @config['region']
+          deploy_struct["region"] = @region
           if @config['flavor'] == "EKS"
             deploy_struct["max_pods"] = @config['kubernetes']['max_pods'].to_s
 # XXX if FargateKS, get the Fargate Profile artifact
@@ -1381,7 +1381,7 @@ start = Time.now
             role["tags"] = cluster["tags"] if !cluster["tags"].nil?
             role["optional_tags"] = cluster["optional_tags"] if !cluster["optional_tags"].nil?
             configurator.insertKitten(role, "roles")
-            MU::Config.addDependency(cluster, cluster["name"]+"pods", "role", phase: "groom")
+            MU::Config.addDependency(cluster, cluster["name"]+"pods", "role", their_phase: "groom")
             if !MU::Master.kubectl
               MU.log "Since I can't find a kubectl executable, you will have to handle all service account, user, and role bindings manually!", MU::WARN
             end
@@ -1530,7 +1530,7 @@ start = Time.now
             role["tags"] = cluster["tags"] if !cluster["tags"].nil?
             role["optional_tags"] = cluster["optional_tags"] if !cluster["optional_tags"].nil?
             configurator.insertKitten(role, "roles")
-            MU::Config.addDependency(cluster, cluster["name"]+"controlplane", "role", phase: "groom")
+            MU::Config.addDependency(cluster, cluster["name"]+"controlplane", "role", their_phase: "groom")
           end
 
           ok
@@ -1580,7 +1580,7 @@ start = Time.now
           @cacert = cloud_desc.certificate_authority.data
           @cluster = @mu_name
           if @config['flavor'] != "Fargate"
-            resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role(role_name: @mu_name+"WORKERS")
+            resp = MU::Cloud::AWS.iam(credentials: @credentials).get_role(role_name: @mu_name+"WORKERS")
             @worker_role_arn = resp.role.arn
           end
           kube_conf = @deploy.deploy_dir+"/kubeconfig-#{@config['name']}"
@@ -1647,7 +1647,7 @@ start = Time.now
               :tags => @tags
             }
             begin
-              resp = MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).describe_fargate_profile(
+              resp = MU::Cloud::AWS.eks(region: @region, credentials: @credentials).describe_fargate_profile(
                 cluster_name: @mu_name,
                 fargate_profile_name: profname
               )
@@ -1660,7 +1660,7 @@ start = Time.now
                 old_desc["subnets"].sort!
                 if !old_desc.eql?(new_desc)
                   MU.log "Deleting Fargate profile #{profname} in order to apply changes", MU::WARN, details: desc 
-                  MU::Cloud::AWS::ContainerCluster.purge_fargate_profile(profname, @mu_name, @config['region'], @credentials)
+                  MU::Cloud::AWS::ContainerCluster.purge_fargate_profile(profname, @mu_name, @region, @credentials)
                 else
                   next
                 end
@@ -1669,9 +1669,9 @@ start = Time.now
               # This is just fine!
             end
             MU.log "Creating EKS Fargate profile #{profname}", details: desc
-            resp = MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).create_fargate_profile(desc)
+            resp = MU::Cloud::AWS.eks(region: @region, credentials: @credentials).create_fargate_profile(desc)
             begin
-              resp = MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).describe_fargate_profile(
+              resp = MU::Cloud::AWS.eks(region: @region, credentials: @credentials).describe_fargate_profile(
                 cluster_name: @mu_name,
                 fargate_profile_name: profname
               )
@@ -1711,19 +1711,19 @@ start = Time.now
             tagme << s.cloud_id
             tagme_elb << s.cloud_id if !s.private?
           }
-          rtbs = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+          rtbs = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_route_tables(
             filters: [ { name: "vpc-id", values: [@vpc.cloud_id] } ]
           ).route_tables
           tagme.concat(rtbs.map { |r| r.route_table_id } )
           main_sg = @deploy.findLitterMate(type: "firewall_rules", name: "server_pool#{@config['name']}workers")
           tagme << main_sg.cloud_id if main_sg
           MU.log "Applying kubernetes.io tags to VPC resources", details: tagme
-          MU::Cloud::AWS.createTag(tagme, "kubernetes.io/cluster/#{@mu_name}", "shared", credentials: @config['credentials'])
-          MU::Cloud::AWS.createTag(tagme_elb, "kubernetes.io/cluster/elb", @mu_name, credentials: @config['credentials'])
+          MU::Cloud::AWS.createTag(tagme, "kubernetes.io/cluster/#{@mu_name}", "shared", credentials: @credentials)
+          MU::Cloud::AWS.createTag(tagme_elb, "kubernetes.io/cluster/elb", @mu_name, credentials: @credentials)
         end
 
         def manage_ecs_workers
-          resp = MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).list_container_instances({
+          resp = MU::Cloud::AWS.ecs(region: @region, credentials: @credentials).list_container_instances({
             cluster: @mu_name
           })
           existing = {}
@@ -1733,7 +1733,7 @@ start = Time.now
               uuids << arn.sub(/^.*?:container-instance\//, "")
             }
             if uuids.size > 0
-              resp = MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).describe_container_instances({
+              resp = MU::Cloud::AWS.ecs(region: @region, credentials: @credentials).describe_container_instances({
                 cluster: @mu_name,
                 container_instances: uuids
               })
@@ -1744,7 +1744,7 @@ start = Time.now
           end
 
           threads = []
-          resource_lookup = MU::Cloud::AWS.listInstanceTypes(@config['region'])[@config['region']]
+          resource_lookup = MU::Cloud::AWS.listInstanceTypes(@region)[@region]
           serverpool = if ['EKS', 'ECS'].include?(@config['flavor'])
             @deploy.findLitterMate(type: "server_pools", name: @config["name"]+"workers")
           end
@@ -1789,7 +1789,7 @@ start = Time.now
                 params[:container_instance_arn] = existing[node.cloud_id].container_instance_arn
                 MU.log "Updating ECS instance #{node} in cluster #{@mu_name}", MU::NOTICE, details: params
               end
-              MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).register_container_instance(params)
+              MU::Cloud::AWS.ecs(region: @region, credentials: @credentials).register_container_instance(params)
 
             }
           }
@@ -1805,7 +1805,7 @@ start = Time.now
             @loadbalancers.each {|lb|
               MU.log "Mapping LB #{lb.mu_name} to service #{c['name']}", MU::INFO
               if lb.cloud_desc.type != "classic"
-                elb_groups = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups({
+                elb_groups = MU::Cloud::AWS.elb2(region: @region, credentials: @credentials).describe_target_groups({
                     load_balancer_arn: lb.cloud_desc.load_balancer_arn
                   })
                   matching_target_groups = []
@@ -1957,7 +1957,7 @@ start = Time.now
           MU.log "Registering task definition #{service_name} with #{container_definitions.size.to_s} containers"
 
 # XXX this helpfully keeps revisions, but let's compare anyway and avoid cluttering with identical ones
-          resp = MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).register_task_definition(task_params)
+          resp = MU::Cloud::AWS.ecs(region: @region, credentials: @credentials).register_task_definition(task_params)
 
           resp.task_definition.task_definition_arn
         end
@@ -1965,7 +1965,7 @@ start = Time.now
         def list_ecs_services
           svc_resp = nil
           MU.retrier([Aws::ECS::Errors::ClusterNotFoundException], wait: 5, max: 10){
-            svc_resp = MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).list_services(
+            svc_resp = MU::Cloud::AWS.ecs(region: @region, credentials: @credentials).list_services(
               cluster: arn
             )
           }
@@ -2005,14 +2005,14 @@ start = Time.now
           if !existing_svcs.include?(service_name)
             MU.log "Creating Service #{service_name}"
 
-            MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).create_service(service_params)
+            MU::Cloud::AWS.ecs(region: @region, credentials: @credentials).create_service(service_params)
           else
             service_params[:service] = service_params[:service_name].dup
             service_params.delete(:service_name)
             service_params.delete(:launch_type)
             MU.log "Updating Service #{service_name}", MU::NOTICE, details: service_params
 
-            MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).update_service(service_params)
+            MU::Cloud::AWS.ecs(region: @region, credentials: @credentials).update_service(service_params)
           end
         end
 

data/modules/mu/providers/aws/database.rb

@@ -224,7 +224,7 @@ module MU
               }
 
               modify_db_cluster_struct[:preferred_maintenance_window] = @config["preferred_maintenance_window"] if @config["preferred_maintenance_window"]
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_cluster(modify_db_cluster_struct)
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_cluster(modify_db_cluster_struct)
               wait_until_available
             end
 
@@ -305,7 +305,7 @@ module MU
         def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
-            "region" => @config['region'],
+            "region" => @region,
             "credentials" => @credentials,
             "cloud_id" => @cloud_id,
           }
@@ -317,8 +317,8 @@ module MU
           end
 
           noun = @config["create_cluster"] ? "cluster" : "db"
-          tags = MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).list_tags_for_resource(
-            resource_name: MU::Cloud::AWS::Database.getARN(@cloud_id, noun, "rds", region: @config['region'], credentials: @credentials)
+          tags = MU::Cloud::AWS.rds(credentials: @credentials, region: @region).list_tags_for_resource(
+            resource_name: MU::Cloud::AWS::Database.getARN(@cloud_id, noun, "rds", region: @region, credentials: @credentials)
           ).tag_list
           if tags and !tags.empty?
             bok['tags'] = MU.structToHash(tags, stringify_keys: true)
@@ -332,11 +332,11 @@ module MU
           bok["create_cluster"] = true if @config['create_cluster']
 
           params = if bok['create_cluster']
-            MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).describe_db_cluster_parameters(
+            MU::Cloud::AWS.rds(credentials: @credentials, region: @region).describe_db_cluster_parameters(
               db_cluster_parameter_group_name: cloud_desc.db_cluster_parameter_group
             ).parameters
           else
-            MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).describe_db_parameters(
+            MU::Cloud::AWS.rds(credentials: @credentials, region: @region).describe_db_parameters(
               db_parameter_group_name: cloud_desc.db_parameter_groups.first.db_parameter_group_name
             ).parameters
           end
@@ -353,7 +353,7 @@ module MU
               id: sg.vpc_security_group_id,
               cloud: "AWS",
               credentials: @credentials,
-              region: @config['region'],
+              region: @region,
               type: "firewall_rules",
             )
           }
@@ -373,7 +373,7 @@ module MU
             # we have no sensible way to handle heterogenous cluster members, so
             # for now just assume they're all the same
             cloud_desc.db_cluster_members.each { |db|
-              member = MU::Cloud::AWS::Database.find(cloud_id: db.db_instance_identifier, region: @config['region'], credentials: @credentials).values.first
+              member = MU::Cloud::AWS::Database.find(cloud_id: db.db_instance_identifier, region: @region, credentials: @credentials).values.first
 
               sizes << member.db_instance_class
               if member.db_subnet_group and member.db_subnet_group.vpc_id
@@ -385,14 +385,14 @@ module MU
             vpcs.uniq!
             bok['size'] = sizes.sort.first if !sizes.empty?
             if !vpcs.empty?
-              myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: vpcs.sort.first.vpc_id, credentials: @credentials, region: @config['region'], dummy_ok: true, no_deploy_search: true).first
+              myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: vpcs.sort.first.vpc_id, credentials: @credentials, region: @region, dummy_ok: true, no_deploy_search: true).first
               bok['vpc'] = myvpc.getReference(vpcs.sort.first.subnets.map { |s| s.subnet_identifier })
             end
           else
             bok['size'] = cloud_desc.db_instance_class
             bok['auto_minor_version_upgrade'] = true if cloud_desc.auto_minor_version_upgrade
             if cloud_desc.db_subnet_group
-              myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: cloud_desc.db_subnet_group.vpc_id, credentials: @credentials, region: @config['region'], dummy_ok: true, no_deploy_search: true).first
+              myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: cloud_desc.db_subnet_group.vpc_id, credentials: @credentials, region: @region, dummy_ok: true, no_deploy_search: true).first
               bok['vpc'] = myvpc.getReference(cloud_desc.db_subnet_group.subnets.map { |s| s.subnet_identifier })
             end
             bok['storage_type'] = cloud_desc.storage_type
@@ -467,13 +467,13 @@ dependencies
             raise MuError, "Couldn't find subnets in #{@vpc} to add to #{@config["subnet_group_name"]}. Make sure the subnets are valid and publicly_accessible is set correctly"
           else
             resp = begin
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_subnet_groups(
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_subnet_groups(
                 db_subnet_group_name: @config["subnet_group_name"]
               )
 # XXX ensure subnet group matches our config?
             rescue ::Aws::RDS::Errors::DBSubnetGroupNotFoundFault
               # Create subnet group
-              resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_subnet_group(
+              resp = MU::Cloud::AWS.rds(region: @region, credentials: @credentials).create_db_subnet_group(
                 db_subnet_group_name: @config["subnet_group_name"],
                 db_subnet_group_description: @config["subnet_group_name"],
                 subnet_ids: subnet_ids,
@@ -511,13 +511,13 @@ dependencies
           if create
             MU.log "Creating a #{cluster ? "cluster" : "database" } parameter group #{@config["parameter_group_name"]}"
 
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send(cluster ? :create_db_cluster_parameter_group : :create_db_parameter_group, params)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send(cluster ? :create_db_cluster_parameter_group : :create_db_parameter_group, params)
           end
 
 
           if @config[fieldname] and !@config[fieldname].empty?
 
-            old_values = MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).send(cluster ? :describe_db_cluster_parameters : :describe_db_parameters, { name_param => @config["parameter_group_name"] } ).parameters
+            old_values = MU::Cloud::AWS.rds(credentials: @credentials, region: @region).send(cluster ? :describe_db_cluster_parameters : :describe_db_parameters, { name_param => @config["parameter_group_name"] } ).parameters
             old_values.map! { |p| [p.parameter_name, p.parameter_value] }.flatten
             old_values = old_values.to_h
 
@@ -532,12 +532,12 @@ dependencies
 
             MU.retrier([Aws::RDS::Errors::InvalidDBParameterGroupState], wait: 30, max: 10) {
               if cluster
-                MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_cluster_parameter_group(
+                MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_cluster_parameter_group(
                   db_cluster_parameter_group_name: @config["parameter_group_name"],
                   parameters: params
                 )
               else
-                MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_parameter_group(
+                MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_parameter_group(
                   db_parameter_group_name: @config["parameter_group_name"],
                   parameters: params
                 )
@@ -586,7 +586,7 @@ dependencies
           if @config["create_cluster"]
             @config['cluster_node_count'] ||= 1
             if @config['cluster_mode'] == "serverless"
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_current_db_cluster_capacity(
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_current_db_cluster_capacity(
                 db_cluster_identifier: @cloud_id,
                 capacity: @config['cluster_node_count']
               )
@@ -612,8 +612,9 @@ dependencies
           if mods.size > 1
             MU.log "Modifying RDS instance #{@cloud_id}", MU::NOTICE, details: mods
             mods[:apply_immediately] = true
+            mods[:allow_major_version_upgrade] = true
             wait_until_available
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @credentials).send("modify_db_#{noun}".to_sym, mods)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("modify_db_#{noun}".to_sym, mods)
             wait_until_available
           end
 
@@ -660,7 +661,7 @@ dependencies
           if !cloud_desc.db_security_groups.empty?
             cloud_desc.db_security_groups.each { |rds_sg|
               begin
-                MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).authorize_db_security_group_ingress(
+                MU::Cloud::AWS.rds(region: @region, credentials: @credentials).authorize_db_security_group_ingress(
                     db_security_group_name: rds_sg.db_security_group_name,
                     cidrip: cidr
                 )
@@ -682,7 +683,7 @@ dependencies
         def notify
           deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true)
           deploy_struct['cloud_id'] = @cloud_id
-          deploy_struct["region"] ||= @config['region']
+          deploy_struct["region"] ||= @region
           deploy_struct["db_name"] ||= @config['db_name']
           deploy_struct
         end
@@ -708,14 +709,14 @@ dependencies
           end
 
           MU.retrier([Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidDBClusterStateFault], wait: 60, max: 10) {
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("create_db_#{@config['create_cluster'] ? "cluster_" : ""}snapshot".to_sym, params)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("create_db_#{@config['create_cluster'] ? "cluster_" : ""}snapshot".to_sym, params)
           }
 
           loop_if = Proc.new {
             if @config["create_cluster"]
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: snap_id).db_cluster_snapshots.first.status != "available"
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: snap_id).db_cluster_snapshots.first.status != "available"
             else
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_snapshots(db_snapshot_identifier: snap_id).db_snapshots.first.status != "available"
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_snapshots(db_snapshot_identifier: snap_id).db_snapshots.first.status != "available"
             end
           }
 
@@ -732,9 +733,9 @@ dependencies
           src_ref = MU::Config::Ref.get(@config["source"])
           resp =
             if @config["create_cluster"]
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: src_ref.id)
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: src_ref.id)
             else
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_snapshots(db_snapshot_identifier: src_ref.id)
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_snapshots(db_snapshot_identifier: src_ref.id)
             end
 
           snapshots = @config["create_cluster"] ? resp.db_cluster_snapshots : resp.db_snapshots
@@ -797,19 +798,22 @@ dependencies
         # @return [void]
         def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {})
 
+          threads = []
+
           ["instance", "cluster"].each { |type|
-            threaded_resource_purge("describe_db_#{type}s".to_sym, "db_#{type}s".to_sym, "db_#{type}_identifier".to_sym, (type == "instance" ? "db" : "cluster"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
+            threads.concat threaded_resource_purge("describe_db_#{type}s".to_sym, "db_#{type}s".to_sym, "db_#{type}_identifier".to_sym, (type == "instance" ? "db" : "cluster"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
               terminate_rds_instance(nil, noop: noop, skipsnapshots: flags["skipsnapshots"], region: region, deploy_id: deploy_id, cloud_id: id, mu_name: id.upcase, credentials: credentials, cluster: (type == "cluster"), known: flags['known'])
   
-            }.each { |t|
-              t.join
             }
           }
+          threads.each { |t|
+            t.join
+          }
 
           threads = threaded_resource_purge(:describe_db_subnet_groups, :db_subnet_groups, :db_subnet_group_name, "subgrp", region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
             MU.log "Deleting RDS subnet group #{id}"
             MU.retrier([Aws::RDS::Errors::InvalidDBSubnetGroupStateFault], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBSubnetGroupNotFoundFault]) {
-              MU::Cloud::AWS.rds(region: region).delete_db_subnet_group(db_subnet_group_name: id) if !noop
+              MU::Cloud::AWS.rds(region: region, credentials: credentials).delete_db_subnet_group(db_subnet_group_name: id) if !noop
             }
           }
 
@@ -817,7 +821,7 @@ dependencies
             threads.concat threaded_resource_purge("describe_#{type}_parameter_groups".to_sym, "#{type}_parameter_groups".to_sym, "#{type}_parameter_group_name".to_sym, (type == "db" ? "pg" : "cluster-pg"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
               MU.log "Deleting RDS #{type} parameter group #{id}"
               MU.retrier([Aws::RDS::Errors::InvalidDBParameterGroupState], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBParameterGroupNotFound]) {
-                MU::Cloud::AWS.rds(region: region).send("delete_#{type}_parameter_group", { "#{type}_parameter_group_name".to_sym => id }) if !noop
+                MU::Cloud::AWS.rds(region: region, credentials: credentials).send("delete_#{type}_parameter_group", { "#{type}_parameter_group_name".to_sym => id }) if !noop
               }
             }
           }
@@ -1259,7 +1263,7 @@ dependencies
         def add_basic
 
           getPassword
-          if @config['source'].nil? or @config['region'] != @config['source'].region
+          if @config['source'].nil? or @region != @config['source'].region
             manageSubnetGroup if @vpc
           else
             MU.log "Note: Read Replicas automatically reside in the same subnet group as the source database, if they're both in the same region. This replica may not land in the VPC you intended.", MU::WARN
@@ -1344,11 +1348,11 @@ dependencies
             if %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
               clean_parent_opts.call
               MU.log "Creating database #{noun} #{@cloud_id} from snapshot #{@config["snapshot_id"]}"
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("restore_db_#{noun}_from_#{noun == "instance" ? "db_" : ""}snapshot".to_sym, params)
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("restore_db_#{noun}_from_#{noun == "instance" ? "db_" : ""}snapshot".to_sym, params)
             else
               clean_parent_opts.call if noun == "instance" and params[:db_cluster_identifier]
-              MU.log "Creating pristine database #{noun} #{@cloud_id} (#{@config['name']}) in #{@config['region']}", MU::NOTICE, details: params
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("create_db_#{noun}".to_sym, params)
+              MU.log "Creating pristine database #{noun} #{@cloud_id} (#{@config['name']}) in #{@region}", MU::NOTICE, details: params
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("create_db_#{noun}".to_sym, params)
             end
           }
         end
@@ -1375,7 +1379,7 @@ dependencies
 
           MU.retrier([Aws::RDS::Errors::InvalidParameterValue], max: 15, wait: 20) {
             MU.log "Creating database #{@config['create_cluster'] ? "cluster" : "instance" } #{@cloud_id} based on point in time backup '#{@config['restore_time']}' of #{@config['source'].id}"
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("restore_db_#{@config['create_cluster'] ? "cluster" : "instance"}_to_point_in_time".to_sym, params)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("restore_db_#{@config['create_cluster'] ? "cluster" : "instance"}_to_point_in_time".to_sym, params)
           }
         end
 
@@ -1396,8 +1400,8 @@ dependencies
             db_subnet_group_name: @config["subnet_group_name"],
             storage_type: @config["storage_type"]
           }
-          if @config["source"].region and @config['region'] != @config["source"].region
-            params[:source_db_instance_identifier] = MU::Cloud::AWS::Database.getARN(@config["source"].id, "db", "rds", region: @config["source"].region, credentials: @config['credentials'])
+          if @config["source"].region and @region != @config["source"].region
+            params[:source_db_instance_identifier] = MU::Cloud::AWS::Database.getARN(@config["source"].id, "db", "rds", region: @config["source"].region, credentials: @credentials)
           end
 
           params[:port] = @config["port"] if @config["port"]
@@ -1412,7 +1416,7 @@ dependencies
 
           MU.retrier([Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidParameterValue, Aws::RDS::Errors::DBSubnetGroupNotAllowedFault], max: 10, wait: 30, on_retry: on_retry) {
             MU.log "Creating read replica database instance #{@cloud_id} for #{@config['source'].id}"
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(params)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).create_db_instance_read_replica(params)
           }
         end
 
@@ -1471,7 +1475,7 @@ dependencies
             end
             mod_config[:vpc_security_group_ids] << localdeploy_rule.cloud_id
 
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance(mod_config)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_instance(mod_config)
             MU.log "Modified database #{@cloud_id} with new security groups: #{mod_config}", MU::NOTICE
           end
 
@@ -1483,7 +1487,7 @@ dependencies
               db_instance_identifier: @cloud_id,
               apply_immediately: true
             }
-            if !@config["read_replica_of"] or @config['region'] == @config['source'].region
+            if !@config["read_replica_of"] or @region == @config['source'].region
               mod_config[:vpc_security_group_ids] = @config["vpc_security_group_ids"]
             end
 
@@ -1500,7 +1504,7 @@ dependencies
               mod_config[:preferred_maintenance_window] = @config["preferred_maintenance_window"]
             end
 
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance(mod_config)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_instance(mod_config)
             wait_until_available
           end
 
@@ -1508,7 +1512,7 @@ dependencies
           if @config['allow_major_version_upgrade'] && @config["creation_style"] == "new"
             MU.log "Setting major database version upgrade on #{@cloud_id}'"
 
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance(
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_instance(
               db_instance_identifier: @cloud_id,
               apply_immediately: true,
               allow_major_version_upgrade: true
@@ -1653,7 +1657,7 @@ dependencies
 
           raise MuError, "terminate_rds_instance requires a non-nil database descriptor (#{cloud_id})" if db.nil? or cloud_id.nil?
 
-          MU.retrier([], wait: 60, loop_if: Proc.new { %w{creating modifying backing-up}.include?(cluster ? db.status : db.db_instance_status) }) {
+          MU.retrier([], wait: 60, loop_if: Proc.new { %w{creating modifying backing-up}.include?(cluster ? db.status : db.db_instance_status) }, loop_msg: "Waiting for RDS #{cluster ? "cluster" : "instance"} #{cloud_id} to be in a valid state for deletion") {
             db = MU::Cloud::AWS::Database.find(cloud_id: cloud_id, region: region, credentials: credentials, cluster: cluster).values.first
             return if db.nil?
           }
@@ -1673,7 +1677,7 @@ dependencies
               params[:skip_final_snapshot] = false
               params[:final_db_snapshot_identifier] = "#{cloud_id}-mufinal"
             end
-
+sleep 30
             if !noop
               on_retry = Proc.new { |e|
                 if [Aws::RDS::Errors::DBSnapshotAlreadyExists, Aws::RDS::Errors::DBClusterSnapshotAlreadyExistsFault, Aws::RDS::Errors::DBClusterQuotaExceeded].include?(e.class)
@@ -1688,7 +1692,7 @@ dependencies
                 end
               }
               del_db = nil
-              MU.retrier([], wait: 10, ignoreme: [Aws::RDS::Errors::DBInstanceNotFound], loop_if: Proc.new { del_db and ((!cluster and del_db.db_instance_status != "deleted") or (cluster and del_db.status != "deleted")) }) {
+              MU.retrier([], wait: 10, ignoreme: [Aws::RDS::Errors::DBInstanceNotFound], loop_if: Proc.new { del_db and ((!cluster and del_db.db_instance_status != "deleted") or (cluster and del_db.status != "deleted")) }, loop_msg: "Waiting for RDS #{cluster ? "cluster" : "instance"} #{cloud_id} to delete") {
                 del_db = MU::Cloud::AWS::Database.find(cloud_id: cloud_id, region: region, cluster: cluster).values.first
               }
             end