cloud-mu 3.2.0 → 3.5.0

data/modules/mu/providers/aws/database.rb

@@ -224,7 +224,7 @@ module MU
               }
 
               modify_db_cluster_struct[:preferred_maintenance_window] = @config["preferred_maintenance_window"] if @config["preferred_maintenance_window"]
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_cluster(modify_db_cluster_struct)
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_cluster(modify_db_cluster_struct)
               wait_until_available
             end
 
@@ -305,7 +305,7 @@ module MU
         def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
-            "region" => @config['region'],
+            "region" => @region,
             "credentials" => @credentials,
             "cloud_id" => @cloud_id,
           }
@@ -317,8 +317,8 @@ module MU
           end
 
           noun = @config["create_cluster"] ? "cluster" : "db"
-          tags = MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).list_tags_for_resource(
-            resource_name: MU::Cloud::AWS::Database.getARN(@cloud_id, noun, "rds", region: @config['region'], credentials: @credentials)
+          tags = MU::Cloud::AWS.rds(credentials: @credentials, region: @region).list_tags_for_resource(
+            resource_name: MU::Cloud::AWS::Database.getARN(@cloud_id, noun, "rds", region: @region, credentials: @credentials)
           ).tag_list
           if tags and !tags.empty?
             bok['tags'] = MU.structToHash(tags, stringify_keys: true)
@@ -332,11 +332,11 @@ module MU
           bok["create_cluster"] = true if @config['create_cluster']
 
           params = if bok['create_cluster']
-            MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).describe_db_cluster_parameters(
+            MU::Cloud::AWS.rds(credentials: @credentials, region: @region).describe_db_cluster_parameters(
               db_cluster_parameter_group_name: cloud_desc.db_cluster_parameter_group
             ).parameters
           else
-            MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).describe_db_parameters(
+            MU::Cloud::AWS.rds(credentials: @credentials, region: @region).describe_db_parameters(
               db_parameter_group_name: cloud_desc.db_parameter_groups.first.db_parameter_group_name
             ).parameters
           end
@@ -353,7 +353,7 @@ module MU
               id: sg.vpc_security_group_id,
               cloud: "AWS",
               credentials: @credentials,
-              region: @config['region'],
+              region: @region,
               type: "firewall_rules",
             )
           }
@@ -373,7 +373,7 @@ module MU
             # we have no sensible way to handle heterogenous cluster members, so
             # for now just assume they're all the same
             cloud_desc.db_cluster_members.each { |db|
-              member = MU::Cloud::AWS::Database.find(cloud_id: db.db_instance_identifier, region: @config['region'], credentials: @credentials).values.first
+              member = MU::Cloud::AWS::Database.find(cloud_id: db.db_instance_identifier, region: @region, credentials: @credentials).values.first
 
               sizes << member.db_instance_class
               if member.db_subnet_group and member.db_subnet_group.vpc_id
@@ -385,14 +385,14 @@ module MU
             vpcs.uniq!
             bok['size'] = sizes.sort.first if !sizes.empty?
             if !vpcs.empty?
-              myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: vpcs.sort.first.vpc_id, credentials: @credentials, region: @config['region'], dummy_ok: true, no_deploy_search: true).first
+              myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: vpcs.sort.first.vpc_id, credentials: @credentials, region: @region, dummy_ok: true, no_deploy_search: true).first
               bok['vpc'] = myvpc.getReference(vpcs.sort.first.subnets.map { |s| s.subnet_identifier })
             end
           else
             bok['size'] = cloud_desc.db_instance_class
             bok['auto_minor_version_upgrade'] = true if cloud_desc.auto_minor_version_upgrade
             if cloud_desc.db_subnet_group
-              myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: cloud_desc.db_subnet_group.vpc_id, credentials: @credentials, region: @config['region'], dummy_ok: true, no_deploy_search: true).first
+              myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: cloud_desc.db_subnet_group.vpc_id, credentials: @credentials, region: @region, dummy_ok: true, no_deploy_search: true).first
               bok['vpc'] = myvpc.getReference(cloud_desc.db_subnet_group.subnets.map { |s| s.subnet_identifier })
             end
             bok['storage_type'] = cloud_desc.storage_type
@@ -467,13 +467,13 @@ dependencies
             raise MuError, "Couldn't find subnets in #{@vpc} to add to #{@config["subnet_group_name"]}. Make sure the subnets are valid and publicly_accessible is set correctly"
           else
             resp = begin
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_subnet_groups(
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_subnet_groups(
                 db_subnet_group_name: @config["subnet_group_name"]
               )
 # XXX ensure subnet group matches our config?
             rescue ::Aws::RDS::Errors::DBSubnetGroupNotFoundFault
               # Create subnet group
-              resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_subnet_group(
+              resp = MU::Cloud::AWS.rds(region: @region, credentials: @credentials).create_db_subnet_group(
                 db_subnet_group_name: @config["subnet_group_name"],
                 db_subnet_group_description: @config["subnet_group_name"],
                 subnet_ids: subnet_ids,
@@ -511,13 +511,13 @@ dependencies
           if create
             MU.log "Creating a #{cluster ? "cluster" : "database" } parameter group #{@config["parameter_group_name"]}"
 
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send(cluster ? :create_db_cluster_parameter_group : :create_db_parameter_group, params)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send(cluster ? :create_db_cluster_parameter_group : :create_db_parameter_group, params)
           end
 
 
           if @config[fieldname] and !@config[fieldname].empty?
 
-            old_values = MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).send(cluster ? :describe_db_cluster_parameters : :describe_db_parameters, { name_param => @config["parameter_group_name"] } ).parameters
+            old_values = MU::Cloud::AWS.rds(credentials: @credentials, region: @region).send(cluster ? :describe_db_cluster_parameters : :describe_db_parameters, { name_param => @config["parameter_group_name"] } ).parameters
             old_values.map! { |p| [p.parameter_name, p.parameter_value] }.flatten
             old_values = old_values.to_h
 
@@ -532,12 +532,12 @@ dependencies
 
             MU.retrier([Aws::RDS::Errors::InvalidDBParameterGroupState], wait: 30, max: 10) {
               if cluster
-                MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_cluster_parameter_group(
+                MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_cluster_parameter_group(
                   db_cluster_parameter_group_name: @config["parameter_group_name"],
                   parameters: params
                 )
               else
-                MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_parameter_group(
+                MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_parameter_group(
                   db_parameter_group_name: @config["parameter_group_name"],
                   parameters: params
                 )
@@ -586,7 +586,7 @@ dependencies
           if @config["create_cluster"]
             @config['cluster_node_count'] ||= 1
             if @config['cluster_mode'] == "serverless"
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_current_db_cluster_capacity(
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_current_db_cluster_capacity(
                 db_cluster_identifier: @cloud_id,
                 capacity: @config['cluster_node_count']
               )
@@ -612,8 +612,9 @@ dependencies
           if mods.size > 1
             MU.log "Modifying RDS instance #{@cloud_id}", MU::NOTICE, details: mods
             mods[:apply_immediately] = true
+            mods[:allow_major_version_upgrade] = true
             wait_until_available
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @credentials).send("modify_db_#{noun}".to_sym, mods)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("modify_db_#{noun}".to_sym, mods)
             wait_until_available
           end
 
@@ -660,7 +661,7 @@ dependencies
           if !cloud_desc.db_security_groups.empty?
             cloud_desc.db_security_groups.each { |rds_sg|
               begin
-                MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).authorize_db_security_group_ingress(
+                MU::Cloud::AWS.rds(region: @region, credentials: @credentials).authorize_db_security_group_ingress(
                     db_security_group_name: rds_sg.db_security_group_name,
                     cidrip: cidr
                 )
@@ -680,9 +681,9 @@ dependencies
         # Return the metadata for this ContainerCluster
         # @return [Hash]
         def notify
-          deploy_struct = MU.structToHash(cloud_desc)
+          deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true)
           deploy_struct['cloud_id'] = @cloud_id
-          deploy_struct["region"] ||= @config['region']
+          deploy_struct["region"] ||= @region
           deploy_struct["db_name"] ||= @config['db_name']
           deploy_struct
         end
@@ -708,14 +709,14 @@ dependencies
           end
 
           MU.retrier([Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidDBClusterStateFault], wait: 60, max: 10) {
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("create_db_#{@config['create_cluster'] ? "cluster_" : ""}snapshot".to_sym, params)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("create_db_#{@config['create_cluster'] ? "cluster_" : ""}snapshot".to_sym, params)
           }
 
           loop_if = Proc.new {
             if @config["create_cluster"]
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: snap_id).db_cluster_snapshots.first.status != "available"
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: snap_id).db_cluster_snapshots.first.status != "available"
             else
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_snapshots(db_snapshot_identifier: snap_id).db_snapshots.first.status != "available"
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_snapshots(db_snapshot_identifier: snap_id).db_snapshots.first.status != "available"
             end
           }
 
@@ -732,9 +733,9 @@ dependencies
           src_ref = MU::Config::Ref.get(@config["source"])
           resp =
             if @config["create_cluster"]
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: src_ref.id)
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: src_ref.id)
             else
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_snapshots(db_snapshot_identifier: src_ref.id)
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_snapshots(db_snapshot_identifier: src_ref.id)
             end
 
           snapshots = @config["create_cluster"] ? resp.db_cluster_snapshots : resp.db_snapshots
@@ -761,7 +762,7 @@ dependencies
         end
 
         # @return [Array<Thread>]
-        def self.threaded_resource_purge(describe_method, list_method, id_method, arn_type, region, credentials, ignoremaster, known: [])
+        def self.threaded_resource_purge(describe_method, list_method, id_method, arn_type, region, credentials, ignoremaster, known: [], deploy_id: MU.deploy_id)
           deletia = []
 
           resp = MU::Cloud::AWS.rds(credentials: credentials, region: region).send(describe_method)
@@ -774,7 +775,7 @@ dependencies
               next
             end
 
-            if should_delete?(tags, resource.send(id_method), ignoremaster, MU.deploy_id, MU.mu_public_ip, known)
+            if should_delete?(tags, resource.send(id_method), ignoremaster, deploy_id, MU.mu_public_ip, known)
               deletia << resource.send(id_method)
             end
           }
@@ -795,29 +796,32 @@ dependencies
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region in which to operate
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {})
+
+          threads = []
 
           ["instance", "cluster"].each { |type|
-            threaded_resource_purge("describe_db_#{type}s".to_sym, "db_#{type}s".to_sym, "db_#{type}_identifier".to_sym, (type == "instance" ? "db" : "cluster"), region, credentials, ignoremaster, known: flags['known']) { |id|
-              terminate_rds_instance(nil, noop: noop, skipsnapshots: flags["skipsnapshots"], region: region, deploy_id: MU.deploy_id, cloud_id: id, mu_name: id.upcase, credentials: credentials, cluster: (type == "cluster"), known: flags['known'])
+            threads.concat threaded_resource_purge("describe_db_#{type}s".to_sym, "db_#{type}s".to_sym, "db_#{type}_identifier".to_sym, (type == "instance" ? "db" : "cluster"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
+              terminate_rds_instance(nil, noop: noop, skipsnapshots: flags["skipsnapshots"], region: region, deploy_id: deploy_id, cloud_id: id, mu_name: id.upcase, credentials: credentials, cluster: (type == "cluster"), known: flags['known'])
   
-            }.each { |t|
-              t.join
             }
           }
+          threads.each { |t|
+            t.join
+          }
 
-          threads = threaded_resource_purge(:describe_db_subnet_groups, :db_subnet_groups, :db_subnet_group_name, "subgrp", region, credentials, ignoremaster, known: flags['known']) { |id|
+          threads = threaded_resource_purge(:describe_db_subnet_groups, :db_subnet_groups, :db_subnet_group_name, "subgrp", region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
             MU.log "Deleting RDS subnet group #{id}"
             MU.retrier([Aws::RDS::Errors::InvalidDBSubnetGroupStateFault], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBSubnetGroupNotFoundFault]) {
-              MU::Cloud::AWS.rds(region: region).delete_db_subnet_group(db_subnet_group_name: id) if !noop
+              MU::Cloud::AWS.rds(region: region, credentials: credentials).delete_db_subnet_group(db_subnet_group_name: id) if !noop
             }
           }
 
           ["db", "db_cluster"].each { |type|
-            threads.concat threaded_resource_purge("describe_#{type}_parameter_groups".to_sym, "#{type}_parameter_groups".to_sym, "#{type}_parameter_group_name".to_sym, (type == "db" ? "pg" : "cluster-pg"), region, credentials, ignoremaster, known: flags['known']) { |id|
+            threads.concat threaded_resource_purge("describe_#{type}_parameter_groups".to_sym, "#{type}_parameter_groups".to_sym, "#{type}_parameter_group_name".to_sym, (type == "db" ? "pg" : "cluster-pg"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
               MU.log "Deleting RDS #{type} parameter group #{id}"
               MU.retrier([Aws::RDS::Errors::InvalidDBParameterGroupState], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBParameterGroupNotFound]) {
-                MU::Cloud::AWS.rds(region: region).send("delete_#{type}_parameter_group", { "#{type}_parameter_group_name".to_sym => id }) if !noop
+                MU::Cloud::AWS.rds(region: region, credentials: credentials).send("delete_#{type}_parameter_group", { "#{type}_parameter_group_name".to_sym => id }) if !noop
               }
             }
           }
@@ -1259,7 +1263,7 @@ dependencies
         def add_basic
 
           getPassword
-          if @config['source'].nil? or @config['region'] != @config['source'].region
+          if @config['source'].nil? or @region != @config['source'].region
             manageSubnetGroup if @vpc
           else
             MU.log "Note: Read Replicas automatically reside in the same subnet group as the source database, if they're both in the same region. This replica may not land in the VPC you intended.", MU::WARN
@@ -1274,7 +1278,7 @@ dependencies
 
 
         def add_cluster_node
-          cluster = MU::Config::Ref.get(@config["member_of_cluster"]).kitten(@deploy, debug: true)
+          cluster = MU::Config::Ref.get(@config["member_of_cluster"]).kitten(@deploy)
           if cluster.nil? or cluster.cloud_id.nil?
             raise MuError.new "Failed to resolve parent cluster of #{@mu_name}", details: @config["member_of_cluster"].to_h
           end
@@ -1344,18 +1348,18 @@ dependencies
             if %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
               clean_parent_opts.call
               MU.log "Creating database #{noun} #{@cloud_id} from snapshot #{@config["snapshot_id"]}"
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("restore_db_#{noun}_from_#{noun == "instance" ? "db_" : ""}snapshot".to_sym, params)
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("restore_db_#{noun}_from_#{noun == "instance" ? "db_" : ""}snapshot".to_sym, params)
             else
               clean_parent_opts.call if noun == "instance" and params[:db_cluster_identifier]
-              MU.log "Creating pristine database #{noun} #{@cloud_id} (#{@config['name']}) in #{@config['region']}", MU::NOTICE, details: params
-              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("create_db_#{noun}".to_sym, params)
+              MU.log "Creating pristine database #{noun} #{@cloud_id} (#{@config['name']}) in #{@region}", MU::NOTICE, details: params
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("create_db_#{noun}".to_sym, params)
             end
           }
         end
 
         # creation_style = point_in_time
         def create_point_in_time
-          @config["source"].kitten(@deploy, debug: true)
+          @config["source"].kitten(@deploy)
           if !@config["source"].id
             raise MuError.new "Database '#{@config['name']}' couldn't resolve cloud id for source database", details: @config["source"].to_h
           end
@@ -1375,13 +1379,13 @@ dependencies
 
           MU.retrier([Aws::RDS::Errors::InvalidParameterValue], max: 15, wait: 20) {
             MU.log "Creating database #{@config['create_cluster'] ? "cluster" : "instance" } #{@cloud_id} based on point in time backup '#{@config['restore_time']}' of #{@config['source'].id}"
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("restore_db_#{@config['create_cluster'] ? "cluster" : "instance"}_to_point_in_time".to_sym, params)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("restore_db_#{@config['create_cluster'] ? "cluster" : "instance"}_to_point_in_time".to_sym, params)
           }
         end
 
         # creation_style = new, existing and read_replica_of is not nil
         def create_read_replica
-          @config["source"].kitten(@deploy, debug: true)
+          @config["source"].kitten(@deploy)
           if !@config["source"].id
             raise MuError.new "Database '#{@config['name']}' couldn't resolve cloud id for source database", details: @config["source"].to_h
           end
@@ -1396,8 +1400,8 @@ dependencies
             db_subnet_group_name: @config["subnet_group_name"],
             storage_type: @config["storage_type"]
           }
-          if @config["source"].region and @config['region'] != @config["source"].region
-            params[:source_db_instance_identifier] = MU::Cloud::AWS::Database.getARN(@config["source"].id, "db", "rds", region: @config["source"].region, credentials: @config['credentials'])
+          if @config["source"].region and @region != @config["source"].region
+            params[:source_db_instance_identifier] = MU::Cloud::AWS::Database.getARN(@config["source"].id, "db", "rds", region: @config["source"].region, credentials: @credentials)
           end
 
           params[:port] = @config["port"] if @config["port"]
@@ -1412,7 +1416,7 @@ dependencies
 
           MU.retrier([Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidParameterValue, Aws::RDS::Errors::DBSubnetGroupNotAllowedFault], max: 10, wait: 30, on_retry: on_retry) {
             MU.log "Creating read replica database instance #{@cloud_id} for #{@config['source'].id}"
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(params)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).create_db_instance_read_replica(params)
           }
         end
 
@@ -1471,7 +1475,7 @@ dependencies
             end
             mod_config[:vpc_security_group_ids] << localdeploy_rule.cloud_id
 
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance(mod_config)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_instance(mod_config)
             MU.log "Modified database #{@cloud_id} with new security groups: #{mod_config}", MU::NOTICE
           end
 
@@ -1483,7 +1487,7 @@ dependencies
               db_instance_identifier: @cloud_id,
               apply_immediately: true
             }
-            if !@config["read_replica_of"] or @config['region'] == @config['source'].region
+            if !@config["read_replica_of"] or @region == @config['source'].region
               mod_config[:vpc_security_group_ids] = @config["vpc_security_group_ids"]
             end
 
@@ -1500,7 +1504,7 @@ dependencies
               mod_config[:preferred_maintenance_window] = @config["preferred_maintenance_window"]
             end
 
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance(mod_config)
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_instance(mod_config)
             wait_until_available
           end
 
@@ -1508,7 +1512,7 @@ dependencies
           if @config['allow_major_version_upgrade'] && @config["creation_style"] == "new"
             MU.log "Setting major database version upgrade on #{@cloud_id}'"
 
-            MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance(
+            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_instance(
               db_instance_identifier: @cloud_id,
               apply_immediately: true,
               allow_major_version_upgrade: true
@@ -1653,7 +1657,7 @@ dependencies
 
           raise MuError, "terminate_rds_instance requires a non-nil database descriptor (#{cloud_id})" if db.nil? or cloud_id.nil?
 
-          MU.retrier([], wait: 60, loop_if: Proc.new { %w{creating modifying backing-up}.include?(cluster ? db.status : db.db_instance_status) }) {
+          MU.retrier([], wait: 60, loop_if: Proc.new { %w{creating modifying backing-up}.include?(cluster ? db.status : db.db_instance_status) }, loop_msg: "Waiting for RDS #{cluster ? "cluster" : "instance"} #{cloud_id} to be in a valid state for deletion") {
             db = MU::Cloud::AWS::Database.find(cloud_id: cloud_id, region: region, credentials: credentials, cluster: cluster).values.first
             return if db.nil?
           }
@@ -1673,7 +1677,7 @@ dependencies
               params[:skip_final_snapshot] = false
               params[:final_db_snapshot_identifier] = "#{cloud_id}-mufinal"
             end
-
+sleep 30
             if !noop
               on_retry = Proc.new { |e|
                 if [Aws::RDS::Errors::DBSnapshotAlreadyExists, Aws::RDS::Errors::DBClusterSnapshotAlreadyExistsFault, Aws::RDS::Errors::DBClusterQuotaExceeded].include?(e.class)
@@ -1688,7 +1692,7 @@ dependencies
                 end
               }
               del_db = nil
-              MU.retrier([], wait: 10, ignoreme: [Aws::RDS::Errors::DBInstanceNotFound], loop_if: Proc.new { del_db and ((!cluster and del_db.db_instance_status != "deleted") or (cluster and del_db.status != "deleted")) }) {
+              MU.retrier([], wait: 10, ignoreme: [Aws::RDS::Errors::DBInstanceNotFound], loop_if: Proc.new { del_db and ((!cluster and del_db.db_instance_status != "deleted") or (cluster and del_db.status != "deleted")) }, loop_msg: "Waiting for RDS #{cluster ? "cluster" : "instance"} #{cloud_id} to delete") {
                 del_db = MU::Cloud::AWS::Database.find(cloud_id: cloud_id, region: region, cluster: cluster).values.first
               }
             end

data/modules/mu/providers/aws/dnszone.rb

@@ -25,7 +25,7 @@ module MU
           super
           @mu_name ||= @deploy.getResourceName(@config["name"])
 
-          MU.setVar("curRegion", @config['region']) if !@config['region'].nil?
+          MU.setVar("curRegion", @region) if !@region.nil?
         end
 
         # Called automatically by {MU::Deploy#createResources}
@@ -42,7 +42,7 @@ module MU
             params = {
               :name => @config['name'],
               :hosted_zone_config => {
-                :comment => MU.deploy_id
+                :comment => @deploy.deploy_id
               },
               :caller_reference => @deploy.getResourceName(@config['name'])
             }
@@ -119,7 +119,7 @@ module MU
                   if @dependencies.has_key?('loadbalancer') and @dependencies['loadbalancer'].has_key?(dnsrec['target']) and !@dependencies['loadbalancer'][dnsrec['target']].cloudobj.nil? and dnsrec['deploy_id'].nil?
                     @dependencies['loadbalancer'][dnsrec['target']].cloudobj.notify['dns']
                   elsif dnsrec['deploy_id']
-                    found = MU::MommaCat.findStray("AWS", "loadbalancer", deploy_id: dnsrec["deploy_id"], mu_name: dnsrec["target"], region: @config["region"])
+                    found = MU::MommaCat.findStray("AWS", "loadbalancer", deploy_id: dnsrec["deploy_id"], mu_name: dnsrec["target"], region: @region)
                     raise MuError, "Couldn't find #{dnsrec['mu_type']} #{dnsrec["target"]}" if found.nil? || found.empty?
                     found.first.deploydata['dns']
                   end
@@ -128,7 +128,7 @@ module MU
                     MU.log "dnsrec['target'] #{dnsrec['target']}"
                     deploydata = @dependencies['server'][dnsrec['target']].deploydata
                   elsif dnsrec['deploy_id']
-                    found = MU::MommaCat.findStray("AWS", "server", deploy_id: dnsrec["deploy_id"], mu_name: dnsrec["target"], region: @config["region"])
+                    found = MU::MommaCat.findStray("AWS", "server", deploy_id: dnsrec["deploy_id"], mu_name: dnsrec["target"], region: @region)
                     raise MuError, "Couldn't find #{dnsrec['mu_type']} #{dnsrec["target"]}" if found.nil? || found.empty?
                     deploydata = found.first.deploydata
                   end
@@ -159,7 +159,7 @@ module MU
                   if @dependencies.has_key?(dnsrec['mu_type']) && dnsrec['deploy_id'].nil?
                     @dependencies[dnsrec['mu_type']][dnsrec['target']].deploydata['endpoint']
                   elsif dnsrec['deploy_id']
-                    found = MU::MommaCat.findStray("AWS", "database", deploy_id: dnsrec["deploy_id"], mu_name: dnsrec["target"], region: @config["region"])
+                    found = MU::MommaCat.findStray("AWS", "database", deploy_id: dnsrec["deploy_id"], mu_name: dnsrec["target"], region: @region)
                     raise MuError, "Couldn't find #{dnsrec['mu_type']} #{dnsrec["target"]}" if found.nil? || found.empty?
                     found.first.deploydata['endpoint']
                   end
@@ -173,11 +173,29 @@ module MU
           return resp.hosted_zone if @config["create_zone"]
         end
 
+        # Resolve a record entry (as in {MU::Config::BasketofKittens::dnszones::records} to the full DNS name we would assign it
+        def self.recordToName(record)
+          shortname = record['name']
+          shortname += ".#{MU.environment.downcase}" if record["append_environment_name"]
+
+          zone = if record['zone'].has_key?("id")
+            MU::Cloud::DNSZone.find(cloud_id: record['zone']['id']).values.first
+          else
+            MU::Cloud::DNSZone.find(cloud_id: record['zone']['name']).values.first
+          end
+
+          if zone.nil?
+            raise MuError.new "Failed to locate Route53 DNS Zone", details: record['zone']
+          end
+
+          shortname+"."+zone.name.sub(/\.$/, '')
+        end
+
         # Wrapper for {MU::Cloud::AWS::DNSZone.manageRecord}. Spawns threads to create all
         # requested records in background and returns immediately.
         # @param cfg [Array]: An array of parsed {MU::Config::BasketofKittens::dnszones::records} objects.
         # @param target [String]: Optional target for the records to be created. Overrides targets embedded in cfg records.
-        def self.createRecordsFromConfig(cfg, target: nil)
+        def self.createRecordsFromConfig(cfg, target: nil, name_only: false)
           return if cfg.nil?
           record_threads = []
 
@@ -190,7 +208,6 @@ module MU
               zone = MU::Cloud::DNSZone.find(cloud_id: record['zone']['name']).values.first
             end
 
-            raise MuError, "Failed to locate Route53 DNS Zone for domain #{record['zone']['name']}" if zone.nil?
 
             healthcheck_id = nil
             record['target'] = target if !target.nil?
@@ -625,7 +642,7 @@ module MU
             # )
             # deploydata.merge!(MU.structToHash(resp.hosted_zone))
             # deploydata['vpcs'] = @config['vpcs'] if !@config['vpcs'].nil?
-            # deploydata["region"] = @config['region'] if !@config['region'].nil?
+            # deploydata["region"] = @region if !@region.nil?
             # @deploy.notify(MU::Cloud::DNSZone.cfg_plural, mu_name, deploydata)
             # return deploydata
 
@@ -666,7 +683,7 @@ module MU
 
         # Called by {MU::Cleanup}. Locates resources that were created by the
         # currently-loaded deployment, and purges them.
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           MU.log "AWS::DNSZone.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           threads = []
@@ -679,7 +696,7 @@ module MU
               muid_match = false
               mumaster_match = false
               tags.each { |tag|
-                muid_match = true if tag.key == "MU-ID" and tag.value == MU.deploy_id
+                muid_match = true if tag.key == "MU-ID" and tag.value == deploy_id
                 mumaster_match = true if tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
               }
 
@@ -723,7 +740,7 @@ module MU
             t.join
           }
 
-          zones = MU::Cloud::DNSZone.find(deploy_id: MU.deploy_id, region: region)
+          zones = MU::Cloud::DNSZone.find(deploy_id: deploy_id, region: region)
           zones.values.each { |zone|
             MU.log "Purging DNS Zone '#{zone.name}' (#{zone.id})"
             if !noop
@@ -779,7 +796,7 @@ module MU
 
               # TO DO: if we have more than one record it will retry the deletion multiple times and will throw Aws::Route53::Errors::InvalidChangeBatch / record not found even though the record was deleted
               zone_rrsets.each { |record|
-                if record.name.match(MU.deploy_id.downcase)
+                if record.name.match(deploy_id.downcase)
                   resource_records = []
                   record.resource_records.each { |rrecord|
                     resource_records << rrecord.value